@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

package/README.md

@@ -1,23 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/byte-utils.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export declare function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export declare function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;

package/byte-utils.js

@@ -0,0 +1,42 @@
+/**
+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.
+ * @ignore
+ */
+export function isStream(value) {
+    if (typeof value !== 'object' || value === null) {
+        return false;
+    }
+    // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.
+    // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would
+    // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.
+    const isNodeStream = typeof value.pipe === 'function' && typeof value[Symbol.asyncIterator] === 'function';
+    const isWebStream = typeof value.pipeTo === 'function';
+    return isNodeStream || isWebStream;
+}
+/**
+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.
+ * @ignore
+ */
+export function isBuffer(value) {
+    return (value != null &&
+        typeof value === 'object' &&
+        (Buffer.isBuffer(value) ||
+            value instanceof ArrayBuffer ||
+            ArrayBuffer.isView(value) ||
+            value.constructor?.name === 'Buffer'));
+}
+/**
+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over
+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned
+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.
+ * @ignore
+ */
+export function toBuffer(value) {
+    if (Buffer.isBuffer(value)) {
+        return value;
+    }
+    if (value instanceof ArrayBuffer) {
+        return Buffer.from(value);
+    }
+    return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+}

package/configuration.d.ts

@@ -1,130 +1,60 @@
-import type { Dictionary } from '@crawlee/types';
-import { LogLevel } from '@apify/log';
-export interface ConfigurationOptions {
-    /**
-     * Default dataset id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_DATASET_ID` environment variable.
-     * @default 'default'
-     */
-    defaultDatasetId?: string;
-    /**
-     * Defines whether to purge the default storage folders before starting the crawler run.
-     *
-     * Alternative to `CRAWLEE_PURGE_ON_START` environment variable.
-     * @default true
-     */
-    purgeOnStart?: boolean;
-    /**
-     * Default key-value store id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
-     * @default 'default'
-     */
-    defaultKeyValueStoreId?: string;
-    /**
-     * Default request queue id.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` environment variable.
-     * @default 'default'
-     */
-    defaultRequestQueueId?: string;
-    /**
-     * Sets the ratio, defining the maximum CPU usage.
-     * When the CPU usage is higher than the provided ratio, the CPU is considered overloaded.
-     * @default 0.95
-     */
-    maxUsedCpuRatio?: number;
-    /**
-     * Sets the ratio, defining the amount of system memory that could be used by the {@link AutoscaledPool}.
-     * When the memory usage is more than the provided ratio, the memory is considered overloaded.
-     *
-     * Alternative to `CRAWLEE_AVAILABLE_MEMORY_RATIO` environment variable.
-     * @default 0.25
-     */
-    availableMemoryRatio?: number;
-    /**
-     * Sets the amount of system memory in megabytes to be used by the {@link AutoscaledPool}.
-     * By default, the maximum memory is set to one quarter of total system memory.
-     *
-     * Alternative to `CRAWLEE_MEMORY_MBYTES` environment variable.
-     */
-    memoryMbytes?: number;
-    /**
-     * Defines the interval of emitting the `persistState` event.
-     *
-     * Alternative to `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable.
-     * @default 60_000
-     */
-    persistStateIntervalMillis?: number;
-    /**
-     Defines the interval of emitting the `systemInfo` event.
-     @default 1_000
-     */
-    systemInfoIntervalMillis?: number;
-    /**
-     * Defines the default input key, i.e. the key that is used to get the crawler input value
-     * from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * Alternative to `CRAWLEE_INPUT_KEY` environment variable.
-     * @default 'INPUT'
-     */
-    inputKey?: string;
-    /**
-     * Defines whether web browsers launched by Crawlee will run in the headless mode.
-     *
-     * Alternative to `CRAWLEE_HEADLESS` environment variable.
-     * @default true
-     */
-    headless?: boolean;
-    /**
-     * Defines whether to run X virtual framebuffer on the web browsers launched by Crawlee.
-     *
-     * Alternative to `CRAWLEE_XVFB` environment variable.
-     * @default false
-     */
-    xvfb?: boolean;
-    /**
-     * Defines a path to Chrome executable.
-     *
-     * Alternative to `CRAWLEE_CHROME_EXECUTABLE_PATH` environment variable.
-     */
-    chromeExecutablePath?: string;
-    /**
-     * Defines a path to default browser executable.
-     *
-     * Alternative to `CRAWLEE_DEFAULT_BROWSER_PATH` environment variable.
-     */
-    defaultBrowserPath?: string;
-    /**
-     * Defines whether to disable browser sandbox by adding `--no-sandbox` flag to `launchOptions`.
-     *
-     * Alternative to `CRAWLEE_DISABLE_BROWSER_SANDBOX` environment variable.
-     */
-    disableBrowserSandbox?: boolean;
-    /**
-     * Sets the log level to the given value.
-     *
-     * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
-     * @default 'INFO'
-     */
-    logLevel?: LogLevel | LogLevel[keyof LogLevel];
-    /**
-     * Defines whether the storage client used should persist the data it stores.
-     *
-     * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
-     */
-    persistStorage?: boolean;
-    /**
-     * Used in place of `isContainerized()` when collecting system metrics.
-     *
-     * Alternative to `CRAWLEE_CONTAINERIZED` environment variable.
-     */
-    containerized?: boolean;
+import { z } from 'zod';
+import { LogLevel } from './log.js';
+export interface ConfigField<T extends z.ZodType = z.ZodType> {
+    schema: T;
+    envVar?: string | string[];
+}
+export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
+/** Zod preprocessor treating `'0'` and `'false'` as falsy. */
+export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;
+export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;
+export declare const crawleeConfigFields: {
+    /** @default true */
+    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    /** @default 0.95 */
+    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    /** @default 0.25 */
+    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    /** @default 60_000 */
+    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    /**
+     * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
+     * the request handler timeout (twice it, and never below 5 minutes).
+     */
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    /** @default 1_000 */
+    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    /** @default true */
+    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    /** @default false */
+    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
+    defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
+    /** @default false */
+    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
+    /** @default true */
+    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    /** @default './storage' */
+    storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
+    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+};
+export type FieldsInput<F extends Record<string, ConfigField>> = {
+    [K in keyof F]?: z.output<F[K]['schema']>;
+};
+export type FieldsOutput<F extends Record<string, ConfigField>> = {
+    [K in keyof F]: z.output<F[K]['schema']>;
+};
+export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;
+export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;
+/** @deprecated Use {@link ConfigurationInput} instead. */
+export type ConfigurationOptions = ConfigurationInput;
+export interface Configuration extends ResolvedConfigValues {
 }
 /**
  * `Configuration` is a value object holding Crawlee configuration. By default, there is a
- * global singleton instance of this class available via `Configuration.getGlobalConfig()`.
+ * global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
  * Places that depend on a configurable behaviour depend on this class, as they have the global
  * instance as the default value.
  *
@@ -133,14 +63,10 @@ export interface ConfigurationOptions {
  * import { BasicCrawler, Configuration } from 'crawlee';
  *
  * // Get the global configuration
- * const config = Configuration.getGlobalConfig();
- * // Set the 'persistStateIntervalMillis' option
- * // of global configuration to 10 seconds
- * config.set('persistStateIntervalMillis', 10_000);
- *
- * // No need to pass the configuration to the crawler,
- * // as it's using the global configuration by default
- * const crawler = new BasicCrawler();
+ * const config = Configuration.getGlobalConfiguration();
+ * // Access configuration values directly as properties
+ * console.log(config.headless);
+ * console.log(config.persistStateIntervalMillis);
  * ```
  *
  * *Using custom configuration:*
@@ -150,15 +76,14 @@ export interface ConfigurationOptions {
  * // Create a new configuration
  * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
  * // Pass the configuration to the crawler
- * const crawler = new BasicCrawler({ configuration: config, ... });
+ * const crawler = new BasicCrawler({ configuration: config });
  * ```
  *
- * The configuration provided via environment variables always takes precedence. We can also
- * define the `crawlee.json` file in the project root directory which will serve as a baseline,
- * so the options provided in constructor will override those. In other words, the precedence is:
+ * Configuration is immutable — values are set via the constructor and cannot be changed afterwards.
+ * The priority order for resolving values is (highest to lowest):
  *
  * ```text
- * crawlee.json < constructor options < environment variables
+ * constructor options > environment variables > crawlee.json > schema defaults
  * ```
  *
  * ## Supported Configuration Options
@@ -168,57 +93,40 @@ export interface ConfigurationOptions {
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
  * `headless` | `CRAWLEE_HEADLESS` | `true`
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
+ * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
  * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
+ * `storageDir` | `CRAWLEE_STORAGE_DIR` | `'./storage'`
  *
  * ## Advanced Configuration Options
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
- * `xvfb` | `CRAWLEE_XVFB` | -
+ * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
  * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
  * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
- * `containerized | `CRAWLEE_CONTAINERIZED | -
+ * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
 export declare class Configuration {
+    #private;
     /**
-     * Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
-     */
-    protected static ENV_MAP: Dictionary;
-    protected static BOOLEAN_VARS: string[];
-    protected static INTEGER_VARS: string[];
-    protected static COMMA_SEPARATED_LIST_VARS: string[];
-    protected static DEFAULTS: Dictionary;
-    protected options: Map<keyof ConfigurationOptions, ConfigurationOptions[keyof ConfigurationOptions]>;
-    /**
-     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
-     */
-    constructor(options?: ConfigurationOptions);
-    /**
-     * Returns configured value. First checks the environment variables, then provided configuration,
-     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
-     * in the above section.
+     * Field definitions for this configuration class.
+     * Subclasses override this to register additional fields.
      */
-    get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
-    protected _castEnvValue(key: keyof ConfigurationOptions, value: number | string | boolean): string | number | boolean | string[];
+    protected static fields: Record<string, ConfigField>;
     /**
-     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
-     * To reset a value, we can omit the `value` argument or pass `undefined` there.
+     * Creates new `Configuration` instance with provided options.
+     * Constructor options take precedence over environment variables, which take precedence
+     * over crawlee.json values, which take precedence over schema defaults.
      */
-    set(key: keyof ConfigurationOptions, value?: any): void;
+    constructor(options?: ConfigurationInput);
     /**
      * Returns the global configuration instance. It will respect the environment variables.
      *
      * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
-    static getGlobalConfig(): Configuration;
-    protected buildOptions(options: ConfigurationOptions): void;
+    static getGlobalConfiguration(): Configuration;
 }
-//# sourceMappingURL=configuration.d.ts.map