@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/configuration.js

@@ -1,15 +1,86 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { MemoryStorage } from '@crawlee/memory-storage';
-import { pathExistsSync } from 'fs-extra/esm';
-import log, { LogLevel } from '@apify/log';
-import { LocalEventManager } from './events/local_event_manager.js';
-import { entries } from './typedefs.js';
+import { z } from 'zod';
+import { log, LogLevel } from './log.js';
+import { serviceLocator } from './service_locator.js';
+// Crawlee attaches many listeners to shared EventEmitters (one per crawler/session/autoscaled pool),
+// which can exceed Node's default limit of 10 and trigger spurious MaxListenersExceededWarning logs.
+// Raising the global default avoids false positives; real leaks will still manifest as unbounded growth.
+// TODO: tracked in https://github.com/apify/crawlee/issues/3615 — find a less side-effecting place for this.
+EventEmitter.defaultMaxListeners = 50;
+export function field(schema, envVar) {
+    return { schema, envVar };
+}
+// --- Zod preprocessors ---
+/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
+export const coerceBoolean = z.preprocess((val) => {
+    if (typeof val === 'string') {
+        return !['0', 'false'].includes(val.toLowerCase());
+    }
+    return val;
+}, z.boolean());
+export const coerceNumber = z.preprocess((val) => {
+    if (typeof val === 'string')
+        return Number(val);
+    return val;
+}, z.number());
+/** Zod schema accepting both LogLevel enum values and string names (case-insensitive). */
+const logLevelSchema = z.preprocess((val) => {
+    if (val == null)
+        return val;
+    const s = String(val);
+    if (Number.isFinite(+s))
+        return +s;
+    const key = s.toUpperCase();
+    if (key in LogLevel)
+        return LogLevel[key];
+    return val;
+}, z.enum(LogLevel));
+// --- Crawlee config field definitions ---
+export const crawleeConfigFields = {
+    /** @default 'default' */
+    defaultDatasetId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_DATASET_ID'),
+    /** @default true */
+    purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
+    /** @default 'default' */
+    defaultKeyValueStoreId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID'),
+    /** @default 'default' */
+    defaultRequestQueueId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID'),
+    /** @default 0.95 */
+    maxUsedCpuRatio: field(coerceNumber.default(0.95)),
+    /** @default 0.25 */
+    availableMemoryRatio: field(coerceNumber.default(0.25), 'CRAWLEE_AVAILABLE_MEMORY_RATIO'),
+    memoryMbytes: field(coerceNumber.optional(), 'CRAWLEE_MEMORY_MBYTES'),
+    /** @default 60_000 */
+    persistStateIntervalMillis: field(coerceNumber.default(60_000), 'CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS'),
+    /**
+     * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
+     * the request handler timeout (twice it, and never below 5 minutes).
+     */
+    internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
+    /** @default 1_000 */
+    systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
+    /** @default 'INPUT' */
+    inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
+    /** @default true */
+    headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
+    /** @default false */
+    xvfb: field(coerceBoolean.default(false), 'CRAWLEE_XVFB'),
+    chromeExecutablePath: field(z.string().optional(), 'CRAWLEE_CHROME_EXECUTABLE_PATH'),
+    defaultBrowserPath: field(z.string().optional(), 'CRAWLEE_DEFAULT_BROWSER_PATH'),
+    /** @default false */
+    disableBrowserSandbox: field(coerceBoolean.default(false), 'CRAWLEE_DISABLE_BROWSER_SANDBOX'),
+    logLevel: field(logLevelSchema.optional(), 'CRAWLEE_LOG_LEVEL'),
+    /** @default true */
+    persistStorage: field(coerceBoolean.default(true), 'CRAWLEE_PERSIST_STORAGE'),
+    /** @default './storage' */
+    storageDir: field(z.string().default('./storage'), 'CRAWLEE_STORAGE_DIR'),
+    containerized: field(coerceBoolean.optional(), 'CRAWLEE_CONTAINERIZED'),
+};
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
- * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
+ * global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
  * Places that depend on a configurable behaviour depend on this class, as they have the global
  * instance as the default value.
  *
@@ -18,14 +89,10 @@ import { entries } from './typedefs.js';
  * import { BasicCrawler, Configuration } from 'crawlee';
  *
  * // Get the global configuration
- * const config = Configuration.getGlobalConfig();
- * // Set the 'persistStateIntervalMillis' option
- * // of global configuration to 10 seconds
- * config.set('persistStateIntervalMillis', 10_000);
- *
- * // No need to pass the configuration to the crawler,
- * // as it's using the global configuration by default
- * const crawler = new BasicCrawler();
+ * const config = Configuration.getGlobalConfiguration();
+ * // Access configuration values directly as properties
+ * console.log(config.headless);
+ * console.log(config.persistStateIntervalMillis);
  * ```
  *
  * *Using custom configuration:*
@@ -35,15 +102,14 @@ import { entries } from './typedefs.js';
  * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
  * // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ ... }, config);
+ * const crawler = new BasicCrawler({ configuration: config });
  * ```
  *
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
+ * The priority order for resolving values is (highest to lowest):
  *
  * ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > schema defaults
  * ```
  *
  * ## Supported Configuration Options
@@ -57,246 +123,128 @@ import { entries } from './typedefs.js';
  * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
  * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
  *
  * ## Advanced Configuration Options
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
  * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
- * `xvfb` | `CRAWLEE_XVFB` | -
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `containerized | `CRAWLEE_CONTAINERIZED | -
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
+// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
 export class Configuration {
     /**
-     * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
-     */
-    static ENV_MAP = {
-        CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
-        CRAWLEE_PURGE_ON_START: 'purgeOnStart',
-        CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
-        CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
-        CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
-        CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
-        CRAWLEE_INPUT_KEY: 'inputKey',
-        CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
-        CRAWLEE_HEADLESS: 'headless',
-        CRAWLEE_XVFB: 'xvfb',
-        CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
-        CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
-        CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
-        CRAWLEE_LOG_LEVEL: 'logLevel',
-        CRAWLEE_PERSIST_STORAGE: 'persistStorage',
-        CRAWLEE_CONTAINERIZED: 'containerized',
-    };
-    static BOOLEAN_VARS = [
-        'purgeOnStart',
-        'headless',
-        'xvfb',
-        'disableBrowserSandbox',
-        'persistStorage',
-        'containerized',
-    ];
-    static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
-    static COMMA_SEPARATED_LIST_VARS = [];
-    static DEFAULTS = {
-        defaultKeyValueStoreId: 'default',
-        defaultDatasetId: 'default',
-        defaultRequestQueueId: 'default',
-        inputKey: 'INPUT',
-        maxUsedCpuRatio: 0.95,
-        availableMemoryRatio: 0.25,
-        storageClientOptions: {},
-        purgeOnStart: true,
-        headless: true,
-        persistStateIntervalMillis: 60_000,
-        systemInfoIntervalMillis: 1_000,
-        persistStorage: true,
-    };
-    /**
-     * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
-     * @internal
+     * Field definitions for this configuration class.
+     * Subclasses override this to register additional fields.
      */
-    static storage = new AsyncLocalStorage();
-    options;
-    services = new Map();
-    /** @internal */
-    static globalConfig;
-    storageManagers = new Map();
+    static fields = crawleeConfigFields;
+    #resolvedValues;
     /**
-     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+     * Creates new `Configuration` instance with provided options.
+     * Constructor options take precedence over environment variables, which take precedence
+     * over crawlee.json values, which take precedence over schema defaults.
      */
     constructor(options = {}) {
-        this.buildOptions(options);
-        // Increase the global limit for event emitter memory leak warnings.
-        EventEmitter.defaultMaxListeners = 50;
-        // set the log level to support CRAWLEE_ prefixed env var too
-        const logLevel = this.get('logLevel');
-        if (logLevel) {
-            const level = Number.isFinite(+logLevel)
-                ? +logLevel
-                : LogLevel[String(logLevel).toUpperCase()];
-            log.setLevel(level);
-        }
-    }
-    /**
-     * Returns configured value. First checks the environment variables, then provided configuration,
-     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
-     * in the above section.
-     */
-    get(key, defaultValue) {
-        // prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
-        let envValue;
-        for (const [k, v] of entries(Configuration.ENV_MAP)) {
-            if (key === v) {
-                envValue = process.env[k];
-                if (envValue) {
-                    break;
-                }
-            }
+        const fields = this.constructor.fields;
+        const fileOptions = Configuration.loadFileOptions();
+        this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
+        this.registerAccessors();
+        // Set the log level
+        const logLevel = this.logLevel;
+        if (logLevel != null) {
+            log.setLevel(logLevel);
         }
-        if (envValue != null) {
-            return this._castEnvValue(key, envValue);
-        }
-        // check instance level options
-        if (this.options.has(key)) {
-            return this.options.get(key);
-        }
-        // fallback to defaults
-        return (defaultValue ?? Configuration.DEFAULTS[key] ?? envValue);
-    }
-    _castEnvValue(key, value) {
-        if (Configuration.INTEGER_VARS.includes(key)) {
-            return +value;
-        }
-        if (Configuration.BOOLEAN_VARS.includes(key)) {
-            // 0, false and empty string are considered falsy values
-            return !['0', 'false', ''].includes(String(value).toLowerCase());
-        }
-        if (Configuration.COMMA_SEPARATED_LIST_VARS.includes(key)) {
-            if (!value)
-                return [];
-            return String(value)
-                .split(',')
-                .map((v) => v.trim());
-        }
-        return value;
     }
     /**
-     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
-     */
-    set(key, value) {
-        this.options.set(key, value);
-    }
-    /**
-     * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
-     */
-    static set(key, value) {
-        this.getGlobalConfig().set(key, value);
-    }
-    /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
-     * return the same client instance.
+     * Returns the global configuration instance. It will respect the environment variables.
      *
-     * Caching works based on the `storageClientOptions`, so calling this method with different options will return
-     * multiple instances, one for each variant of the options.
-     * @internal
+     * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
-    getStorageClient() {
-        if (this.options.has('storageClient')) {
-            return this.options.get('storageClient');
-        }
-        const options = this.options.get('storageClientOptions');
-        return this.createMemoryStorage(options);
-    }
-    getEventManager() {
-        if (this.options.has('eventManager')) {
-            return this.options.get('eventManager');
-        }
-        if (this.services.has('eventManager')) {
-            return this.services.get('eventManager');
-        }
-        const eventManager = new LocalEventManager(this);
-        this.services.set('eventManager', eventManager);
-        return eventManager;
+    static getGlobalConfiguration() {
+        return serviceLocator.getConfiguration();
     }
     /**
-     * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
-     * @internal
+     * Resolves all field values once using the priority chain:
+     * constructor options > env vars > crawlee.json > schema defaults.
      */
-    createMemoryStorage(options = {}) {
-        const cacheKey = `MemoryStorage-${JSON.stringify(options)}`;
-        if (this.services.has(cacheKey)) {
-            return this.services.get(cacheKey);
+    static resolveAll(fields, userOptions, fileOptions) {
+        const values = {};
+        for (const [key, fieldDef] of Object.entries(fields)) {
+            // 1. Constructor options (highest priority)
+            if (key in userOptions && userOptions[key] !== undefined) {
+                values[key] = fieldDef.schema.parse(userOptions[key]);
+                continue;
+            }
+            // 2. Environment variables
+            const envValue = Configuration.readEnvVar(fieldDef);
+            if (envValue != null) {
+                values[key] = fieldDef.schema.parse(envValue);
+                continue;
+            }
+            // 3. crawlee.json file options
+            if (key in fileOptions && fileOptions[key] !== undefined) {
+                values[key] = fieldDef.schema.parse(fileOptions[key]);
+                continue;
+            }
+            // 4. Schema default (by parsing undefined through the schema)
+            const parsed = fieldDef.schema.safeParse(undefined);
+            values[key] = parsed.success ? parsed.data : undefined;
         }
-        const storage = new MemoryStorage({
-            persistStorage: this.get('persistStorage'),
-            // Override persistStorage if user provides it via storageClientOptions
-            ...options,
-        });
-        this.services.set(cacheKey, storage);
-        return storage;
-    }
-    useStorageClient(client) {
-        this.options.set('storageClient', client);
-    }
-    static useStorageClient(client) {
-        this.getGlobalConfig().useStorageClient(client);
-    }
-    useEventManager(events) {
-        this.options.set('eventManager', events);
+        return values;
     }
     /**
-     * Returns the global configuration instance. It will respect the environment variables.
+     * Registers getters (and throwing setters) on the instance for each field.
      */
-    static getGlobalConfig() {
-        if (Configuration.storage.getStore()) {
-            return Configuration.storage.getStore();
+    registerAccessors() {
+        const fields = this.constructor.fields;
+        const descriptors = {};
+        for (const key of Object.keys(fields)) {
+            descriptors[key] = {
+                get: () => this.#resolvedValues[key],
+                set() {
+                    throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
+                },
+                enumerable: true,
+                configurable: false,
+            };
         }
-        Configuration.globalConfig ??= new Configuration();
-        return Configuration.globalConfig;
-    }
-    /**
-     * Gets default {@link StorageClient} instance.
-     */
-    static getStorageClient() {
-        return this.getGlobalConfig().getStorageClient();
+        Object.defineProperties(this, descriptors);
     }
     /**
-     * Gets default {@link EventManager} instance.
+     * Reads the first defined env var value for a field definition.
+     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
+     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static getEventManager() {
-        return this.getGlobalConfig().getEventManager();
+    static readEnvVar(fieldDef) {
+        if (!fieldDef.envVar)
+            return undefined;
+        const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
+        for (const envVar of envVars) {
+            const value = process.env[envVar];
+            if (value != null && value !== '')
+                return value;
+        }
+        return undefined;
     }
     /**
-     * Resets global configuration instance. The default instance holds configuration based on env vars,
-     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     * Loads config options from crawlee.json in the current working directory.
      */
-    static resetGlobalState() {
-        delete this.globalConfig;
-    }
-    buildOptions(options) {
-        // try to load configuration from crawlee.json as the baseline
-        const path = join(process.cwd(), 'crawlee.json');
-        if (pathExistsSync(path)) {
-            try {
-                const file = readFileSync(path);
-                const optionsFromFileConfig = JSON.parse(file.toString());
-                Object.assign(options, optionsFromFileConfig);
-            }
-            catch {
-                // ignore
-            }
+    static loadFileOptions() {
+        try {
+            const file = readFileSync(join(process.cwd(), 'crawlee.json'));
+            return JSON.parse(file.toString());
+        }
+        catch {
+            return {};
         }
-        this.options = new Map(entries(options));
     }
 }
-//# sourceMappingURL=configuration.js.map

package/cookie_utils.d.ts

@@ -25,9 +25,11 @@ export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): Coo
 /**
  * Transforms browser-pool cookie to tough-cookie.
  * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
  * @internal
  */
-export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number): Cookie;
+export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number): Cookie;
 /**
  * @internal
  * @param cookieString The cookie string to attempt parsing
@@ -40,4 +42,3 @@ export declare function cookieStringToToughCookie(cookieString: string): CookieO
  * @internal
  */
 export declare function mergeCookies(url: string, sourceCookies: string[]): string;
-//# sourceMappingURL=cookie_utils.d.ts.map

package/cookie_utils.js

@@ -1,5 +1,5 @@
 import { Cookie, CookieJar } from 'tough-cookie';
-import { log } from './log.js';
+import { serviceLocator } from './service_locator.js';
 import { CookieParseError } from './session_pool/errors.js';
 /**
  * @internal
@@ -47,19 +47,25 @@ export function toughCookieToBrowserPoolCookie(toughCookie) {
 /**
  * Transforms browser-pool cookie to tough-cookie.
  * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
  * @internal
  */
 export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
     const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
-    const expires = isExpiresValid
-        ? new Date(cookieObject.expires * 1000)
-        : getDefaultCookieExpirationDate(maxAgeSecs);
+    let expires;
+    if (isExpiresValid) {
+        expires = new Date(cookieObject.expires * 1000);
+    }
+    else if (maxAgeSecs != null) {
+        expires = getDefaultCookieExpirationDate(maxAgeSecs);
+    }
     const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
     const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
     return new Cookie({
         key: cookieObject.name,
         value: cookieObject.value,
-        expires,
+        ...(expires !== undefined && { expires }),
         domain,
         path: cookieObject.path,
         secure: cookieObject.secure,
@@ -97,15 +103,20 @@ export function mergeCookies(url, sourceCookies) {
             if (!cookieString)
                 continue;
             const cookie = Cookie.parse(cookieString);
+            if (!cookie) {
+                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
+                continue;
+            }
             const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
                 return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
             });
             if (similarKeyCookie) {
-                log.warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
             }
             jar.setCookieSync(cookie, url);
         }
     }
     return jar.getCookieStringSync(url);
 }
-//# sourceMappingURL=cookie_utils.js.map

package/crawlers/context_pipeline.d.ts

@@ -40,6 +40,16 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
      * @returns A new ContextPipeline instance with the added middleware
      */
     abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+    /**
+     * Chains another pipeline onto this one. The other pipeline's base context must match
+     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
+     * middlewares first, then the other pipeline's middlewares.
+     *
+     * @template TFinalContext - The final context type after the chained pipeline's transformations
+     * @param other - The pipeline to append after this one
+     * @returns A new ContextPipeline combining both pipelines' middlewares
+     */
+    abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
     /**
      * Executes the middleware pipeline and passes the final context to a consumer function.
      *
@@ -58,4 +68,3 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
      */
     abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
 }
-//# sourceMappingURL=context_pipeline.d.ts.map

package/crawlers/context_pipeline.js

@@ -1,4 +1,5 @@
 import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
+import { serviceLocator } from '../service_locator.js';
 /**
  * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
  *
@@ -25,12 +26,12 @@ export class ContextPipeline {
  * properties from the `ContextPipeline` interface, making type checking more reliable.
  */
 class ContextPipelineImpl extends ContextPipeline {
-    middleware;
-    parent;
+    #middleware;
+    #parent;
     constructor(middleware, parent) {
         super();
-        this.middleware = middleware;
-        this.parent = parent;
+        this.#middleware = middleware;
+        this.#parent = parent;
     }
     /**
      * @inheritdoc
@@ -38,11 +39,19 @@ class ContextPipelineImpl extends ContextPipeline {
     compose(middleware) {
         return new ContextPipelineImpl(middleware, this);
     }
+    chain(other) {
+        const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
+        let result = this;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware);
+        }
+        return result;
+    }
     *middlewareChain() {
         let step = this;
         while (step !== undefined) {
-            yield step.middleware;
-            step = step.parent;
+            yield step.#middleware;
+            step = step.#parent;
         }
     }
     /**
@@ -56,7 +65,22 @@ class ContextPipelineImpl extends ContextPipeline {
             for (const { action, cleanup } of middlewares) {
                 try {
                     const contextExtension = await action(crawlingContext);
-                    Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));
+                    const extensionNames = [
+                        ...Object.getOwnPropertyNames(contextExtension),
+                        ...Object.getOwnPropertySymbols(contextExtension),
+                    ];
+                    for (const key of extensionNames) {
+                        try {
+                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
+                            }
+                        }
+                        catch (error) {
+                            serviceLocator
+                                .getLogger()
+                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                        }
+                    }
                     if (cleanup) {
                         cleanupStack.push(cleanup);
                     }
@@ -96,4 +120,3 @@ class ContextPipelineImpl extends ContextPipeline {
         }
     }
 }
-//# sourceMappingURL=context_pipeline.js.map