@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/configuration.js

@@ -1,12 +1,78 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { MemoryStorage } from '@crawlee/memory-storage';
-import { pathExistsSync } from 'fs-extra/esm';
-import log, { LogLevel } from '@apify/log';
-import { LocalEventManager } from './events/local_event_manager.js';
-import { entries } from './typedefs.js';
+import { z } from 'zod';
+import { log, LogLevel } from './log.js';
+import { serviceLocator } from './service_locator.js';
+// Crawlee attaches many listeners to shared EventEmitters (one per crawler/session/autoscaled pool),
+// which can exceed Node's default limit of 10 and trigger spurious MaxListenersExceededWarning logs.
+// Raising the global default avoids false positives; real leaks will still manifest as unbounded growth.
+// TODO: tracked in https://github.com/apify/crawlee/issues/3615 — find a less side-effecting place for this.
+EventEmitter.defaultMaxListeners = 50;
+export function field(schema, envVar) {
+    return { schema, envVar };
+}
+// --- Zod preprocessors ---
+/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
+export const coerceBoolean = z.preprocess((val) => {
+    if (typeof val === 'string') {
+        return !['0', 'false'].includes(val.toLowerCase());
+    }
+    return val;
+}, z.boolean());
+export const coerceNumber = z.preprocess((val) => {
+    if (typeof val === 'string')
+        return Number(val);
+    return val;
+}, z.number());
+/** Zod schema accepting both LogLevel enum values and string names (case-insensitive). */
+const logLevelSchema = z.preprocess((val) => {
+    if (val == null)
+        return val;
+    const s = String(val);
+    if (Number.isFinite(+s))
+        return +s;
+    const key = s.toUpperCase();
+    if (key in LogLevel)
+        return LogLevel[key];
+    return val;
+}, z.nativeEnum(LogLevel));
+// --- Crawlee config field definitions ---
+export const crawleeConfigFields = {
+    /** @default 'default' */
+    defaultDatasetId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_DATASET_ID'),
+    /** @default true */
+    purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
+    /** @default 'default' */
+    defaultKeyValueStoreId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID'),
+    /** @default 'default' */
+    defaultRequestQueueId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID'),
+    /** @default 0.95 */
+    maxUsedCpuRatio: field(coerceNumber.default(0.95)),
+    /** @default 0.25 */
+    availableMemoryRatio: field(coerceNumber.default(0.25), 'CRAWLEE_AVAILABLE_MEMORY_RATIO'),
+    memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
+    /** @default 60_000 */
+    persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
+    /** @default 1_000 */
+    systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
+    /** @default 'INPUT' */
+    inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
+    /** @default true */
+    headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
+    /** @default false */
+    xvfb: field(coerceBoolean.default(false), 'CRAWLEE_XVFB'),
+    chromeExecutablePath: field(z.string().optional(), 'CRAWLEE_CHROME_EXECUTABLE_PATH'),
+    defaultBrowserPath: field(z.string().optional(), 'CRAWLEE_DEFAULT_BROWSER_PATH'),
+    /** @default false */
+    disableBrowserSandbox: field(coerceBoolean.default(false), 'CRAWLEE_DISABLE_BROWSER_SANDBOX'),
+    logLevel: field(logLevelSchema.optional(), 'CRAWLEE_LOG_LEVEL'),
+    /** @default true */
+    persistStorage: field(coerceBoolean.default(true), 'CRAWLEE_PERSIST_STORAGE'),
+    /** @default './storage' */
+    storageDir: field(z.string().default('./storage'), 'CRAWLEE_STORAGE_DIR'),
+    containerized: field(coerceBoolean.optional(), 'CRAWLEE_CONTAINERIZED'),
+};
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
  * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
@@ -19,13 +85,9 @@ import { entries } from './typedefs.js';
  *
  * // Get the global configuration
  * const config = Configuration.getGlobalConfig();
- * // Set the 'persistStateIntervalMillis' option
- * // of global configuration to 10 seconds
- * config.set('persistStateIntervalMillis', 10_000);
- *
- * // No need to pass the configuration to the crawler,
- * // as it's using the global configuration by default
- * const crawler = new BasicCrawler();
+ * // Access configuration values directly as properties
+ * console.log(config.headless);
+ * console.log(config.persistStateIntervalMillis);
  * ```
  *
  * *Using custom configuration:*
@@ -35,15 +97,14 @@ import { entries } from './typedefs.js';
  * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
  * // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ ... }, config);
+ * const crawler = new BasicCrawler({ configuration: config });
  * ```
  *
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
+ * The priority order for resolving values is (highest to lowest):
  *
  * ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > schema defaults
  * ```
  *
  * ## Supported Configuration Options
@@ -59,248 +120,125 @@ import { entries } from './typedefs.js';
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
  *
  * ## Advanced Configuration Options
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
  * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
- * `xvfb` | `CRAWLEE_XVFB` | -
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
- * `containerized | `CRAWLEE_CONTAINERIZED | -
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
+// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
 export class Configuration {
     /**
-     * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
-     */
-    static ENV_MAP = {
-        CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
-        CRAWLEE_PURGE_ON_START: 'purgeOnStart',
-        CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
-        CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
-        CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
-        CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
-        CRAWLEE_INPUT_KEY: 'inputKey',
-        CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
-        CRAWLEE_HEADLESS: 'headless',
-        CRAWLEE_XVFB: 'xvfb',
-        CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
-        CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
-        CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
-        CRAWLEE_LOG_LEVEL: 'logLevel',
-        CRAWLEE_PERSIST_STORAGE: 'persistStorage',
-        CRAWLEE_SYSTEM_INFO_V2: 'systemInfoV2',
-        CRAWLEE_CONTAINERIZED: 'containerized',
-    };
-    static BOOLEAN_VARS = [
-        'purgeOnStart',
-        'headless',
-        'xvfb',
-        'disableBrowserSandbox',
-        'persistStorage',
-        'systemInfoV2',
-        'containerized',
-    ];
-    static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
-    static COMMA_SEPARATED_LIST_VARS = [];
-    static DEFAULTS = {
-        defaultKeyValueStoreId: 'default',
-        defaultDatasetId: 'default',
-        defaultRequestQueueId: 'default',
-        inputKey: 'INPUT',
-        maxUsedCpuRatio: 0.95,
-        availableMemoryRatio: 0.25,
-        storageClientOptions: {},
-        purgeOnStart: true,
-        headless: true,
-        persistStateIntervalMillis: 60_000,
-        systemInfoIntervalMillis: 1_000,
-        persistStorage: true,
-        systemInfoV2: false,
-    };
-    /**
-     * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
-     * @internal
+     * Field definitions for this configuration class.
+     * Subclasses override this to register additional fields.
      */
-    static storage = new AsyncLocalStorage();
-    options;
-    services = new Map();
-    /** @internal */
-    static globalConfig;
-    storageManagers = new Map();
+    static fields = crawleeConfigFields;
+    resolvedValues;
     /**
-     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+     * Creates new `Configuration` instance with provided options.
+     * Constructor options take precedence over environment variables, which take precedence
+     * over crawlee.json values, which take precedence over schema defaults.
      */
     constructor(options = {}) {
-        this.buildOptions(options);
-        // Increase the global limit for event emitter memory leak warnings.
-        EventEmitter.defaultMaxListeners = 50;
-        // set the log level to support CRAWLEE_ prefixed env var too
-        const logLevel = this.get('logLevel');
-        if (logLevel) {
-            const level = Number.isFinite(+logLevel)
-                ? +logLevel
-                : LogLevel[String(logLevel).toUpperCase()];
-            log.setLevel(level);
-        }
-    }
-    /**
-     * Returns configured value. First checks the environment variables, then provided configuration,
-     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
-     * in the above section.
-     */
-    get(key, defaultValue) {
-        // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
-        let envValue;
-        for (const [k, v] of entries(Configuration.ENV_MAP)) {
-            if (key === v) {
-                envValue = process.env[k];
-                if (envValue) {
-                    break;
-                }
-            }
-        }
-        if (envValue != null) {
-            return this._castEnvValue(key, envValue);
-        }
-        // check instance level options
-        if (this.options.has(key)) {
-            return this.options.get(key);
+        const fields = this.constructor.fields;
+        const fileOptions = Configuration.loadFileOptions();
+        this.resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
+        this.registerAccessors();
+        // Set the log level
+        const logLevel = this.logLevel;
+        if (logLevel != null) {
+            log.setLevel(logLevel);
         }
-        // fallback to defaults
-        return (defaultValue ?? Configuration.DEFAULTS[key] ?? envValue);
-    }
-    _castEnvValue(key, value) {
-        if (Configuration.INTEGER_VARS.includes(key)) {
-            return +value;
-        }
-        if (Configuration.BOOLEAN_VARS.includes(key)) {
-            // 0, false and empty string are considered falsy values
-            return !['0', 'false', ''].includes(String(value).toLowerCase());
-        }
-        if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
-            if (!value)
-                return [];
-            return String(value)
-                .split(',')
-                .map((v) => v.trim());
-        }
-        return value;
     }
     /**
-     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
-     */
-    set(key, value) {
-        this.options.set(key, value);
-    }
-    /**
-     * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
-     */
-    static set(key, value) {
-        this.getGlobalConfig().set(key, value);
-    }
-    /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
-     * return the same client instance.
+     * Returns the global configuration instance. It will respect the environment variables.
      *
-     * Caching works based on the `storageClientOptions`, so calling this method with different options will return
-     * multiple instances, one for each variant of the options.
-     * @internal
+     * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
-    getStorageClient() {
-        if (this.options.has('storageClient')) {
-            return this.options.get('storageClient');
-        }
-        const options = this.options.get('storageClientOptions');
-        return this.createMemoryStorage(options);
-    }
-    getEventManager() {
-        if (this.options.has('eventManager')) {
-            return this.options.get('eventManager');
-        }
-        if (this.services.has('eventManager')) {
-            return this.services.get('eventManager');
-        }
-        const eventManager = new LocalEventManager(this);
-        this.services.set('eventManager', eventManager);
-        return eventManager;
+    static getGlobalConfig() {
+        return serviceLocator.getConfiguration();
     }
     /**
-     * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
-     * @internal
+     * Resolves all field values once using the priority chain:
+     * constructor options > env vars > crawlee.json > schema defaults.
      */
-    createMemoryStorage(options = {}) {
-        const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
-        if (this.services.has(cacheKey)) {
-            return this.services.get(cacheKey);
+    static resolveAll(fields, userOptions, fileOptions) {
+        const values = {};
+        for (const [key, fieldDef] of Object.entries(fields)) {
+            // 1. Constructor options (highest priority)
+            if (key in userOptions && userOptions[key] !== undefined) {
+                values[key] = fieldDef.schema.parse(userOptions[key]);
+                continue;
+            }
+            // 2. Environment variables
+            const envValue = Configuration.readEnvVar(fieldDef);
+            if (envValue != null) {
+                values[key] = fieldDef.schema.parse(envValue);
+                continue;
+            }
+            // 3. crawlee.json file options
+            if (key in fileOptions && fileOptions[key] !== undefined) {
+                values[key] = fieldDef.schema.parse(fileOptions[key]);
+                continue;
+            }
+            // 4. Schema default (by parsing undefined through the schema)
+            const result = fieldDef.schema.safeParse(undefined);
+            values[key] = result.success ? result.data : undefined;
         }
-        const storage = new MemoryStorage({
-            persistStorage: this.get('persistStorage'),
-            // Override persistStorage if user provides it via storageClientOptions
-            ...options,
-        });
-        this.services.set(cacheKey, storage);
-        return storage;
-    }
-    useStorageClient(client) {
-        this.options.set('storageClient', client);
-    }
-    static useStorageClient(client) {
-        this.getGlobalConfig().useStorageClient(client);
-    }
-    useEventManager(events) {
-        this.options.set('eventManager', events);
+        return values;
     }
     /**
-     * Returns the global configuration instance. It will respect the environment variables.
+     * Registers getters (and throwing setters) on the instance for each field.
      */
-    static getGlobalConfig() {
-        if (Configuration.storage.getStore()) {
-            return Configuration.storage.getStore();
+    registerAccessors() {
+        const fields = this.constructor.fields;
+        const descriptors = {};
+        for (const key of Object.keys(fields)) {
+            descriptors[key] = {
+                get: () => this.resolvedValues[key],
+                set() {
+                    throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
+                },
+                enumerable: true,
+                configurable: false,
+            };
         }
-        Configuration.globalConfig ??= new Configuration();
-        return Configuration.globalConfig;
-    }
-    /**
-     * Gets default {@link StorageClient} instance.
-     */
-    static getStorageClient() {
-        return this.getGlobalConfig().getStorageClient();
+        Object.defineProperties(this, descriptors);
     }
     /**
-     * Gets default {@link EventManager} instance.
+     * Reads the first defined env var value for a field definition.
+     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
+     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static getEventManager() {
-        return this.getGlobalConfig().getEventManager();
+    static readEnvVar(fieldDef) {
+        if (!fieldDef.envVar)
+            return undefined;
+        const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
+        for (const envVar of envVars) {
+            const value = process.env[envVar];
+            if (value != null && value !== '')
+                return value;
+        }
+        return undefined;
     }
     /**
-     * Resets global configuration instance. The default instance holds configuration based on env vars,
-     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     * Loads config options from crawlee.json in the current working directory.
      */
-    static resetGlobalState() {
-        delete this.globalConfig;
-    }
-    buildOptions(options) {
-        // try to load configuration from crawlee.json as the baseline
-        const path = join(process.cwd(), 'crawlee.json');
-        if (pathExistsSync(path)) {
-            try {
-                const file = readFileSync(path);
-                const optionsFromFileConfig = JSON.parse(file.toString());
-                Object.assign(options, optionsFromFileConfig);
-            }
-            catch {
-                // ignore
-            }
+    static loadFileOptions() {
+        try {
+            const file = readFileSync(join(process.cwd(), 'crawlee.json'));
+            return JSON.parse(file.toString());
+        }
+        catch {
+            return {};
         }
-        this.options = new Map(entries(options));
     }
 }
-//# sourceMappingURL=configuration.js.map

package/cookie_utils.d.ts

@@ -7,7 +7,7 @@ export interface ResponseLike {
 /**
  * @internal
  */
-export declare function getCookiesFromResponse(response: ResponseLike): Cookie[];
+export declare function getCookiesFromResponse(response: Response): Cookie[];
 /**
  * Calculate cookie expiration date
  * @param maxAgeSecs
@@ -25,9 +25,11 @@ export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): Coo
 /**
  * Transforms browser-pool cookie to tough-cookie.
  * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
  * @internal
  */
-export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number): Cookie;
+export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number): Cookie;
 /**
  * @internal
  * @param cookieString The cookie string to attempt parsing
@@ -40,4 +42,3 @@ export declare function cookieStringToToughCookie(cookieString: string): CookieO
  * @internal
  */
 export declare function mergeCookies(url: string, sourceCookies: string[]): string;
-//# sourceMappingURL=cookie_utils.d.ts.map

package/cookie_utils.js

@@ -1,19 +1,17 @@
 import { Cookie, CookieJar } from 'tough-cookie';
-import { log } from './log.js';
+import { serviceLocator } from './service_locator.js';
 import { CookieParseError } from './session_pool/errors.js';
 /**
  * @internal
  */
 export function getCookiesFromResponse(response) {
-    const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
-    const cookieHeader = headers?.['set-cookie'] || '';
+    const headers = response.headers;
+    const cookieHeaders = headers.getSetCookie();
     try {
-        return Array.isArray(cookieHeader)
-            ? cookieHeader.map((cookie) => Cookie.parse(cookie))
-            : [Cookie.parse(cookieHeader)];
+        return cookieHeaders.map((cookie) => Cookie.parse(cookie));
     }
     catch (e) {
-        throw new CookieParseError(cookieHeader);
+        throw new CookieParseError(cookieHeaders);
     }
 }
 /**
@@ -49,19 +47,25 @@ export function toughCookieToBrowserPoolCookie(toughCookie) {
 /**
  * Transforms browser-pool cookie to tough-cookie.
  * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
  * @internal
  */
 export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
     const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
-    const expires = isExpiresValid
-        ? new Date(cookieObject.expires * 1000)
-        : getDefaultCookieExpirationDate(maxAgeSecs);
+    let expires;
+    if (isExpiresValid) {
+        expires = new Date(cookieObject.expires * 1000);
+    }
+    else if (maxAgeSecs != null) {
+        expires = getDefaultCookieExpirationDate(maxAgeSecs);
+    }
     const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
     const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
     return new Cookie({
         key: cookieObject.name,
         value: cookieObject.value,
-        expires,
+        ...(expires !== undefined && { expires }),
         domain,
         path: cookieObject.path,
         secure: cookieObject.secure,
@@ -99,15 +103,18 @@ export function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = Cookie.parse(cookieString);
+            if (!cookie)
+                throw new CookieParseError(cookieString);
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });
             if (similarKeyCookie) {
-                log.warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
             }
             jar.setCookieSync(cookie, url);
         }
     }
     return jar.getCookieStringSync(url);
 }
-//# sourceMappingURL=cookie_utils.js.map

package/crawlers/context_pipeline.d.ts

@@ -0,0 +1,70 @@
+import type { Awaitable } from '@crawlee/types';
+/**
+ * Represents a middleware step in the context pipeline.
+ *
+ * @template TCrawlingContext - The input context type for this middleware
+ * @template TCrawlingContextExtension - The enhanced output context type
+ */
+export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
+    /** The main middleware function that enhances the context */
+    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
+    /** Optional cleanup function called after the consumer finishes or fails */
+    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
+}
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
+    /**
+     * Adds a middleware to the pipeline, creating a new pipeline instance.
+     *
+     * This method provides a fluent interface for building context transformation pipelines.
+     * Each middleware can enhance the context with additional properties or utilities.
+     *
+     * @template TCrawlingContextExtension - The enhanced context type produced by this middleware
+     * @param middleware - The middleware to add to the pipeline
+     * @returns A new ContextPipeline instance with the added middleware
+     */
+    abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+    /**
+     * Chains another pipeline onto this one. The other pipeline's base context must match
+     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
+     * middlewares first, then the other pipeline's middlewares.
+     *
+     * @template TFinalContext - The final context type after the chained pipeline's transformations
+     * @param other - The pipeline to append after this one
+     * @returns A new ContextPipeline combining both pipelines' middlewares
+     */
+    abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
+    /**
+     * Executes the middleware pipeline and passes the final context to a consumer function.
+     *
+     * This method runs the crawling context through the entire middleware chain, enhancing it
+     * at each step, and then passes the final enhanced context to the provided consumer function.
+     * Proper cleanup is performed even if exceptions occur during processing.
+     *
+     * @param crawlingContext - The initial context to process through the pipeline
+     * @param finalContextConsumer - The function that will receive the final enhanced context
+     *
+     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
+     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
+     * @throws {RequestHandlerError} When the final context consumer throws an exception
+     * @throws {ContextPipelineCleanupError} When cleanup operations fail
+     * @throws {SessionError} Session errors are re-thrown as-is for special handling
+     */
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
+}