@crawlee/core 4.0.0-beta.27 → 4.0.0-beta.271

package/package.json

@@ -1,13 +1,14 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.27",
+    "version": "4.0.0-beta.271",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -38,7 +39,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -47,27 +48,26 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/consts": "^2.41.0",
-        "@apify/datastructures": "^2.0.3",
-        "@apify/log": "^2.5.18",
-        "@apify/pseudo_url": "^2.0.59",
-        "@apify/timeout": "^0.3.2",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/memory-storage": "4.0.0-beta.27",
-        "@crawlee/types": "4.0.0-beta.27",
-        "@crawlee/utils": "4.0.0-beta.27",
+        "@apify/consts": "^3.0.1",
+        "@apify/datastructures": "^3.0.1",
+        "@apify/log": "^3.0.1",
+        "@apify/timeout": "^1.0.1",
+        "@apify/utilities": "^3.1.0",
+        "@crawlee/fs-storage": "4.0.0-beta.271",
+        "@crawlee/http-client": "4.0.0-beta.271",
+        "@crawlee/types": "4.0.0-beta.271",
+        "@crawlee/utils": "4.0.0-beta.271",
         "@sapphire/async-queue": "^1.5.5",
-        "@vladfrangu/async_event_emitter": "^2.4.6",
-        "csv-stringify": "^6.5.2",
-        "fs-extra": "^11.3.0",
+        "@standard-schema/spec": "^1.1.0",
+        "@vladfrangu/async_event_emitter": "^2.4.7",
+        "content-type": "^3.1.1",
+        "csv-stringify": "^6.9.0",
         "json5": "^2.2.3",
-        "minimatch": "^10.0.1",
-        "ow": "^2.0.0",
-        "stream-json": "^1.9.1",
-        "tldts": "^7.0.6",
-        "tough-cookie": "^6.0.0",
-        "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "mime-types": "^3.0.2",
+        "stream-json": "^3.8.0",
+        "tslib": "^2.9.1",
+        "type-fest": "^4.41.0",
+        "zod": "^4.6.7"
     },
     "lerna": {
         "command": {
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "9d140cc3d54fd41b95cac5aec50eb0767a7480f7"
+    "gitHead": "49e67f1babcf06460323acae1039eb432fc43a67"
 }

package/proxy-configuration.d.ts

@@ -0,0 +1,118 @@
+import type { ProxyInfo } from '@crawlee/types';
+export interface ProxyConfigurationFunction {
+    (): string | null | Promise<string | null>;
+}
+export interface ProxyConfigurationOptions {
+    /**
+     * An array of custom proxy URLs to be rotated.
+     * Custom proxies are not compatible with Apify Proxy and an attempt to use both
+     * configuration options will cause an error to be thrown on initialize.
+     */
+    proxyUrls?: (string | null)[];
+    /**
+     * Custom function that allows you to generate the new proxy URL dynamically.
+     * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
+     *
+     * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
+     */
+    newUrlFunction?: ProxyConfigurationFunction;
+    /**
+     * When truthy, the constructor throws unless one of `proxyUrls` / `newUrlFunction` was given. Falsy by
+     * default, so a bare `ProxyConfiguration` can be constructed. Set by the Apify SDK, which builds the options
+     * object itself; declared here only so that it stays type-checkable.
+     * @internal
+     */
+    validateRequired?: boolean;
+}
+/**
+ * Minimal contract that any object passed to a crawler as its `proxyConfiguration`
+ * option must satisfy.
+ *
+ * Implement this interface to plug a custom proxy-provisioning strategy into any Crawlee
+ * crawler — for example a remote proxy service or a thin wrapper around the built-in
+ * `ProxyConfiguration` with different rotation rules. *
+ *
+ * @category Scaling
+ */
+export interface IProxyConfiguration {
+    /**
+     * Creates a new {@link ProxyInfo} object describing the proxy to use.
+     * Returns `undefined` when no proxy should be used.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations
+     *  should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when
+     *  there is nothing to refresh.
+     */
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
+}
+/**
+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
+ *
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
+ * be rotated by the configuration if this option is provided.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ *
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'],
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   requestHandler({ proxyInfo }) {
+ *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+ *   }
+ * })
+ *
+ * ```
+ * @category Scaling
+ */
+export declare class ProxyConfiguration implements IProxyConfiguration {
+    #private;
+    readonly isManInTheMiddle = false;
+    /**
+     * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
+     * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+     * them to use the selected proxies for all connections.
+     *
+     * ```javascript
+     * const proxyConfiguration = new ProxyConfiguration({
+     *     proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
+     * });
+     *
+     * const crawler = new CheerioCrawler({
+     *   // ...
+     *   proxyConfiguration,
+     *   requestHandler({ proxyInfo }) {
+     *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+     *   }
+     * })
+     *
+     * ```
+     */
+    constructor(options?: ProxyConfigurationOptions);
+    /**
+     * This function creates a new {@link ProxyInfo} info object.
+     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
+     * the currently used proxy via the requestHandler parameter `proxyInfo`.
+     * Use it if you want to work with a rich representation of a proxy URL.
+     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
+     * @return Represents information about used proxy and its configuration.
+     */
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
+    /**
+     * Returns a new proxy URL based on provided configuration options.
+     *
+     * @return A string with a proxy URL, including authentication credentials and port number.
+     *  For example, `http://bob:password123@proxy.example.com:8000`
+     */
+    newUrl(): Promise<string | undefined>;
+}

package/proxy-configuration.js

@@ -0,0 +1,141 @@
+import { z } from 'zod';
+import { parseArgument, schemas } from './validators.js';
+const proxyConfigurationOptionsSchema = z.strictObject({
+    proxyUrls: z
+        .array(z.union([z.url(), z.null()]))
+        .nonempty()
+        .optional(),
+    newUrlFunction: schemas.anyFunction.optional(),
+});
+/**
+ * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
+ * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+ * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
+ * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
+ *
+ * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
+ * be rotated by the configuration if this option is provided.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ *
+ * const proxyConfiguration = new ProxyConfiguration({
+ *   proxyUrls: ['...', '...'],
+ * });
+ *
+ * const crawler = new CheerioCrawler({
+ *   // ...
+ *   proxyConfiguration,
+ *   requestHandler({ proxyInfo }) {
+ *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+ *   }
+ * })
+ *
+ * ```
+ * @category Scaling
+ */
+export class ProxyConfiguration {
+    isManInTheMiddle = false;
+    #nextCustomUrlIndex = 0;
+    #proxyUrls;
+    #newUrlFunction;
+    /**
+     * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
+     * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
+     * them to use the selected proxies for all connections.
+     *
+     * ```javascript
+     * const proxyConfiguration = new ProxyConfiguration({
+     *     proxyUrls: ['http://user:pass@proxy-1.com', 'http://user:pass@proxy-2.com'],
+     * });
+     *
+     * const crawler = new CheerioCrawler({
+     *   // ...
+     *   proxyConfiguration,
+     *   requestHandler({ proxyInfo }) {
+     *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
+     *   }
+     * })
+     *
+     * ```
+     */
+    constructor(options = {}) {
+        // `validateRequired` is destructured off before the strict-object parse on purpose: the Apify SDK passes it
+        // through a computed key (`['validateRequired' as string]: false`), and leaving it in `rest` would make
+        // `Actor.createProxyConfiguration()` fail the `z.strictObject` check with a `ZodError`.
+        const { validateRequired, ...rest } = options;
+        if ('tieredProxyUrls' in rest) {
+            throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
+                'See the v4 upgrading guide for the recommended migration to named sessions.');
+        }
+        const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
+        if (proxyUrls && newUrlFunction)
+            this.#throwCannotCombineCustomMethods();
+        if (!proxyUrls && !newUrlFunction && validateRequired)
+            this.#throwNoOptionsProvided();
+        this.#proxyUrls = proxyUrls;
+        this.#newUrlFunction = newUrlFunction;
+    }
+    /**
+     * This function creates a new {@link ProxyInfo} info object.
+     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
+     * the currently used proxy via the requestHandler parameter `proxyInfo`.
+     * Use it if you want to work with a rich representation of a proxy URL.
+     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
+     * @return Represents information about used proxy and its configuration.
+     */
+    async newProxyInfo(proxyInfo) {
+        if (proxyInfo)
+            return proxyInfo;
+        const url = await this.newUrl();
+        if (!url)
+            return undefined;
+        const { username, password, port, hostname } = new URL(url);
+        return {
+            url,
+            username: decodeURIComponent(username),
+            password: decodeURIComponent(password),
+            hostname,
+            port: port,
+        };
+    }
+    /**
+     * Returns a new proxy URL based on provided configuration options.
+     *
+     * @return A string with a proxy URL, including authentication credentials and port number.
+     *  For example, `http://bob:password123@proxy.example.com:8000`
+     */
+    async newUrl() {
+        if (this.#newUrlFunction) {
+            return (await this.#callNewUrlFunction()) ?? undefined;
+        }
+        return this.#handleProxyUrlsList() ?? undefined;
+    }
+    #handleProxyUrlsList() {
+        return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
+    }
+    /**
+     * Calls the custom newUrlFunction and checks format of its return value
+     */
+    async #callNewUrlFunction() {
+        const proxyUrl = await this.#newUrlFunction();
+        try {
+            if (proxyUrl) {
+                new URL(proxyUrl); // eslint-disable-line no-new
+            }
+            return proxyUrl;
+        }
+        catch (err) {
+            throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
+        }
+    }
+    #throwCannotCombineCustomMethods() {
+        throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
+    }
+    #throwNoOptionsProvided() {
+        throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
+    }
+}

package/recoverable-state.d.ts

@@ -0,0 +1,177 @@
+import type { Configuration } from './configuration.js';
+import type { CrawleeLogger } from './log.js';
+import { KeyValueStore } from './storages/key-value-store.js';
+import type { Awaitable } from '@crawlee/types';
+import type { StandardSchemaV1 } from '@standard-schema/spec';
+/**
+ * One direction of the conversion between the state model and its persisted form - either a plain function, or a
+ * [Standard Schema](https://standardschema.dev) whose validated output is the result.
+ *
+ * A schema that fails to validate makes {@link RecoverableState} throw a {@link StateValidationError}. Zod
+ * codecs work directly, as their validation *is* the decode direction; use `(state) => codec.encode(state)` for the
+ * other one.
+ */
+export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * A {@link StateConversion} for a caller that cannot await one - {@link Statistics}, whose `toJSON()` is
+ * synchronous, being the reason this exists.
+ *
+ * Only the function arm can be narrowed here: a Standard Schema is free to validate asynchronously, so a schema
+ * that does is rejected when it runs rather than when it is passed.
+ */
+export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;
+/**
+ * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
+ * the value.
+ *
+ * @internal
+ */
+export declare function convertStateSync<TFrom, TTo>(conversion: SyncStateConversion<TFrom, TTo>, value: TFrom, persistStateKey: string): TTo;
+export interface RecoverableStatePersistenceOptions {
+    /**
+     * The key under which the state is stored in the KeyValueStore
+     */
+    persistStateKey: string;
+    /**
+     * Flag to enable or disable state persistence
+     */
+    persistenceEnabled?: boolean;
+    /**
+     * The KeyValueStore to persist into, defaulting to the default store. Accepts a pending
+     * {@link KeyValueStore.open} so that callers do not have to be async to point at a specific store.
+     */
+    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
+    /**
+     * Time limit for a single load or save of the state, in milliseconds.
+     * @default 60_000
+     */
+    persistenceTimeoutMillis?: number;
+}
+/**
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
+ */
+export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
+    /**
+     * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
+     *
+     * A plain value is deep-copied with `structuredClone` each time it is used, so pass a factory for a state
+     * that `structuredClone` cannot rebuild - one holding class instances, say, or one derived from a schema.
+     */
+    defaultState: TStateModel | (() => TStateModel);
+    /**
+     * A logger instance for logging operations related to state persistence
+     */
+    logger?: CrawleeLogger;
+    /**
+     * Configuration instance to use when opening the KeyValueStore
+     */
+    configuration?: Configuration;
+    /**
+     * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
+     * If not provided, the state is persisted as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
+     * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
+     * stream.
+     */
+    serialize?: StateConversion<TStateModel, TPersistedState>;
+    /**
+     * Optional conversion of a persisted value back to the state model, and the place to validate a record before
+     * trusting it. If not provided, the persisted value is used as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
+     * instead of a parsed value.
+     */
+    deserialize?: StateConversion<TPersistedState, TStateModel>;
+    /**
+     * Content type of the persisted record. Setting it hands the record encoding over to
+     * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
+     * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
+     * for `JSON.stringify`, which `serialize` can then stream out instead.
+     */
+    contentType?: string;
+}
+/**
+ * Options for configuring the RecoverableState
+ */
+export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
+    contentType?: undefined;
+} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
+/**
+ * A class for managing persistent recoverable state using a plain JavaScript object.
+ *
+ * This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
+ * across migrations or restarts. It manages the loading, saving, and resetting of state data,
+ * with optional persistence capabilities.
+ *
+ * The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
+ * The class automatically hooks into the event system to persist state when needed.
+ */
+export declare class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
+    #private;
+    /**
+     * Initialize a new recoverable state object.
+     *
+     * @param options Configuration options for the recoverable state
+     */
+    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
+    /**
+     * Initialize the recoverable state.
+     *
+     * If persistence is enabled, this method loads the saved state and registers the object to listen for
+     * PERSIST_STATE events. A state established beforehand by {@link RecoverableState.reset} survives if there
+     * is no record to restore.
+     *
+     * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
+     *
+     * @returns The loaded state object
+     */
+    initialize(): Promise<TStateModel>;
+    /**
+     * Clean up resources used by the recoverable state.
+     *
+     * If persistence is enabled, this method deregisters the object from PERSIST_STATE events
+     * and persists the current state one last time, warning rather than throwing if that write fails - cleanup
+     * runs when the work is already done, and failing it would bury whatever the caller was doing. The in-memory
+     * state is left alone, and {@link RecoverableState.initialize} can be called again to open a new
+     * persistence window.
+     */
+    teardown(): Promise<void>;
+    /**
+     * Get the current state.
+     *
+     * Throws until the state has been established, by either {@link RecoverableState.initialize} or the
+     * synchronous {@link RecoverableState.reset} - the latter being how a caller that cannot await in its
+     * constructor gets a usable state right away.
+     */
+    get currentValue(): TStateModel;
+    /**
+     * Reset the in-memory state to the default values, leaving any persisted record alone.
+     *
+     * Use {@link RecoverableState.resetStore} to clear the persisted record as well.
+     */
+    reset(): void;
+    /**
+     * Clear the persisted state record, leaving the in-memory state alone.
+     *
+     * This is a between-lifecycles operation - its point is to stop the next {@link RecoverableState.initialize}
+     * from restoring the record, so it throws while PERSIST_STATE events are still being handled, where the next
+     * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
+     * or {@link RecoverableState.teardown} before clearing the record.
+     *
+     * A no-op if persistence is disabled.
+     */
+    resetStore(): Promise<void>;
+    /**
+     * Persist the current state to the KeyValueStore.
+     *
+     * This method is typically called in response to a PERSIST_STATE event, but can also be called
+     * directly when needed. It is a no-op if persistence is disabled, if no KeyValueStore is available yet, or if
+     * there is no state to write. A failed write only rejects here - the periodic and teardown ones warn instead.
+     *
+     * @param eventData Optional data associated with a PERSIST_STATE event
+     */
+    persistState(eventData?: Record<string, unknown>): Promise<void>;
+}