@crawlee/core 4.0.0-beta.1 → 4.0.0-beta.100

package/configuration.d.ts

@@ -1,157 +1,68 @@
-import { AsyncLocalStorage } from 'node:async_hooks';
-import type { MemoryStorageOptions } from '@crawlee/memory-storage';
-import { MemoryStorage } from '@crawlee/memory-storage';
-import type { Dictionary, StorageClient } from '@crawlee/types';
-import { LogLevel } from '@apify/log';
-import { type EventManager } from './events/event_manager.js';
-import type { StorageManager } from './storages/storage_manager.js';
-import { type Constructor } from './typedefs.js';
-export interface ConfigurationOptions {
-    /**
-     * Defines storage client to be used.
-     * @default {@link MemoryStorage}
-     */
-    storageClient?: StorageClient;
-    /**
-     * Defines the Event Manager to be used.
-     * @default {@link EventManager}
-     */
-    eventManager?: EventManager;
-    /**
-     * Could be used to adjust the storage client behavior
-     * e.g. {@link MemoryStorageOptions} could be used to adjust the {@link MemoryStorage} behavior.
-     */
-    storageClientOptions?: Dictionary;
-    /**
-     * Default dataset id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
-     * @default 'default'
-     */
-    defaultDatasetId?: string;
-    /**
-     * Defines whether to purge the default storage folders before starting the crawler run.
-     *
-     * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
-     * @default true
-     */
-    purgeOnStart?: boolean;
-    /**
-     * Default key-value store id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
-     * @default 'default'
-     */
-    defaultKeyValueStoreId?: string;
-    /**
-     * Default request queue id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
-     * @default 'default'
-     */
-    defaultRequestQueueId?: string;
-    /**
-     * Sets the ratio, defining the maximum CPU usage.
-     * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
-     * @default 0.95
-     */
-    maxUsedCpuRatio?: number;
-    /**
-     * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
-     * When the memory usage is more than the provided ratio, the memory is considered overloaded.
-     *
-     * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
-     * @default 0.25
-     */
-    availableMemoryRatio?: number;
-    /**
-     * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
-     * By default, the maximum memory is set to one quarter of total system memory.
-     *
-     * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
-     */
-    memoryMbytes?: number;
-    /**
-     * Defines the interval of emitting the `persistState` event.
-     *
-     * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
-     * @default 60_000
-     */
-    persistStateIntervalMillis?: number;
-    /**
-     Defines the interval of emitting the `systemInfo` event.
-     @default 1_000
-     */
-    systemInfoIntervalMillis?: number;
-    /**
-     * Defines the default input key, i.e. the key that is used to get the crawler input value
-     * from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
-     * @default 'INPUT'
-     */
-    inputKey?: string;
-    /**
-     * Defines whether web browsers launched by Crawlee will run in the headless mode.
-     *
-     * Alternative to `CRAWLEE_HEADLESS` environment variable.
-     * @default true
-     */
-    headless?: boolean;
-    /**
-     * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
-     *
-     * Alternative to `CRAWLEE_XVFB` environment variable.
-     * @default false
-     */
-    xvfb?: boolean;
-    /**
-     * Defines a path to Chrome executable.
-     *
-     * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
-     */
-    chromeExecutablePath?: string;
-    /**
-     * Defines a path to default browser executable.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
-     */
-    defaultBrowserPath?: string;
-    /**
-     * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
-     *
-     * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
-     */
-    disableBrowserSandbox?: boolean;
-    /**
-     * Sets the log level to the given value.
-     *
-     * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
-     * @default 'INFO'
-     */
-    logLevel?: LogLevel | LogLevel[keyof LogLevel];
-    /**
-     * Defines whether the storage client used should persist the data it stores.
-     *
-     * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
-     */
-    persistStorage?: boolean;
-    /**
-     * Defines whether to use the systemInfoV2 metric collection experiment.
-     *
-     * Alternative to `CRAWLEE_SYSTEM_INFO_V2` environment variable.
-     */
-    systemInfoV2?: boolean;
-    /**
-     * Used in place of `isContainerized()` when collecting system metrics.
-     *
-     * Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
-     */
-    containerized?: boolean;
+import { z } from 'zod';
+import { LogLevel } from './log.js';
+export interface ConfigField<T extends z.ZodType = z.ZodType> {
+    schema: T;
+    envVar?: string | string[];
+}
+export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
+/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
+export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
+export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
+export declare const crawleeConfigFields: {
+    /** @default 'default' */
+    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
+    /** @default true */
+    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    /** @default 'default' */
+    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
+    /** @default 'default' */
+    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
+    /** @default 0.95 */
+    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /** @default 0.25 */
+    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /** @default 60_000 */
+    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /**
+     * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
+     * the request handler timeout (twice it, and never below 5 minutes).
+     */
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /** @default 1_000 */
+    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    /** @default 'INPUT' */
+    inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
+    /** @default true */
+    headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    /** @default false */
+    xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
+    defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
+    /** @default false */
+    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
+    /** @default true */
+    persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    /** @default './storage' */
+    storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
+    containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+};
+export type FieldsInput<F extends Record<string, ConfigField>> = {
+    [K in keyof F]?: z.output<F[K]['schema']>;
+};
+export type FieldsOutput<F extends Record<string, ConfigField>> = {
+    [K in keyof F]: z.output<F[K]['schema']>;
+};
+export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
+export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
+/** @deprecated Use {@link ConfigurationInput} instead. */
+export type ConfigurationOptions = ConfigurationInput;
+export interface Configuration extends ResolvedConfigValues {
 }
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
- * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
+ * global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
  * Places that depend on a configurable behaviour depend on this class, as they have the global
  * instance as the default value.
  *
@@ -160,14 +71,10 @@ export interface ConfigurationOptions {
  * import { BasicCrawler, Configuration } from 'crawlee';
  *
  * // Get the global configuration
- * const config = Configuration.getGlobalConfig();
- * // Set the 'persistStateIntervalMillis' option
- * // of global configuration to 10 seconds
- * config.set('persistStateIntervalMillis', 10_000);
- *
- * // No need to pass the configuration to the crawler,
- * // as it's using the global configuration by default
- * const crawler = new BasicCrawler();
+ * const config = Configuration.getGlobalConfiguration();
+ * // Access configuration values directly as properties
+ * console.log(config.headless);
+ * console.log(config.persistStateIntervalMillis);
  * ```
  *
  * *Using custom configuration:*
@@ -177,15 +84,14 @@ export interface ConfigurationOptions {
  * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
  * // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ ... }, config);
+ * const crawler = new BasicCrawler({ configuration: config });
  * ```
  *
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
+ * The priority order for resolving values is (highest to lowest):
  *
  * ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > schema defaults
  * ```
  *
  * ## Supported Configuration Options
@@ -199,99 +105,59 @@ export interface ConfigurationOptions {
  * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
  * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
  *
  * ## Advanced Configuration Options
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
  * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
- * `xvfb` | `CRAWLEE_XVFB` | -
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
- * `containerized | `CRAWLEE_CONTAINERIZED | -
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
 export declare class Configuration {
     /**
-     * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
-     */
-    protected static ENV_MAP: Dictionary;
-    protected static BOOLEAN_VARS: string[];
-    protected static INTEGER_VARS: string[];
-    protected static COMMA_SEPARATED_LIST_VARS: string[];
-    protected static DEFAULTS: Dictionary;
-    /**
-     * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
-     * @internal
-     */
-    static storage: AsyncLocalStorage<Configuration>;
-    protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
-    protected services: Map<string, unknown>;
-    /** @internal */
-    static globalConfig?: Configuration;
-// @ts-ignore optional peer dependency or compatibility with es2022
-    readonly storageManagers: Map<Constructor, StorageManager<import("./storages/storage_manager.js").IStorage>>;
-    /**
-     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
-     */
-    constructor(options?: ConfigurationOptions);
-    /**
-     * Returns configured value. First checks the environment variables, then provided configuration,
-     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
-     * in the above section.
-     */
-    get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
-    protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean | string[];
-    /**
-     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
+     * Field definitions for this configuration class.
+     * Subclasses override this to register additional fields.
      */
-    set(key: keyof ConfigurationOptions, value?: any): void;
+    protected static fields: Record<string, ConfigField>;
+    private resolvedValues;
     /**
-     * Sets value for given option. Only affects the global `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
+     * Creates new `Configuration` instance with provided options.
+     * Constructor options take precedence over environment variables, which take precedence
+     * over crawlee.json values, which take precedence over schema defaults.
      */
-    static set(key: keyof ConfigurationOptions, value?: any): void;
+    constructor(options?: ConfigurationInput);
     /**
-     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
-     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
-     * return the same client instance.
+     * Returns the global configuration instance. It will respect the environment variables.
      *
-     * Caching works based on the `storageClientOptions`, so calling this method with different options will return
-     * multiple instances, one for each variant of the options.
-     * @internal
-     */
-    getStorageClient(): StorageClient;
-    getEventManager(): EventManager;
-    /**
-     * Creates an instance of MemoryStorage using options as defined in the environment variables or in this `Configuration` instance.
-     * @internal
+     * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
-    createMemoryStorage(options?: MemoryStorageOptions): MemoryStorage;
-    useStorageClient(client: StorageClient): void;
-    static useStorageClient(client: StorageClient): void;
-    useEventManager(events: EventManager): void;
+    static getGlobalConfiguration(): Configuration;
     /**
-     * Returns the global configuration instance. It will respect the environment variables.
+     * Resolves all field values once using the priority chain:
+     * constructor options > env vars > crawlee.json > schema defaults.
      */
-    static getGlobalConfig(): Configuration;
+    private static resolveAll;
     /**
-     * Gets default {@link StorageClient} instance.
+     * Registers getters (and throwing setters) on the instance for each field.
      */
-    static getStorageClient(): StorageClient;
+    private registerAccessors;
     /**
-     * Gets default {@link EventManager} instance.
+     * Reads the first defined env var value for a field definition.
+     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
+     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static getEventManager(): EventManager;
+    private static readEnvVar;
     /**
-     * Resets global configuration instance. The default instance holds configuration based on env vars,
-     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     * Loads config options from crawlee.json in the current working directory.
      */
-    static resetGlobalState(): void;
-    protected buildOptions(options: ConfigurationOptions): void;
+    private static loadFileOptions;
 }
-//# sourceMappingURL=configuration.d.ts.map