@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/configuration.d.ts

@@ -6,47 +6,39 @@ export interface ConfigField<T extends z.ZodType = z.ZodType> {
 }
 export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
 /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
-export declare const coerceBoolean: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>;
-export declare const coerceNumber: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>;
+export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;
+export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;
 export declare const crawleeConfigFields: {
-    /** @default 'default' */
-    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default true */
-    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
-    /** @default 'default' */
-    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
-    /** @default 'default' */
-    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
+    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default 0.95 */
-    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 0.25 */
-    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
-    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 60_000 */
-    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /**
      * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
      * the request handler timeout (twice it, and never below 5 minutes).
      */
-    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 1_000 */
-    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
-    /** @default 'INPUT' */
-    inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
+    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default true */
-    headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default false */
-    xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
     defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
     /** @default false */
-    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
-    logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof LogLevel>>>>;
+    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
     /** @default true */
-    persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default './storage' */
     storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
-    containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
+    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
 };
 export type FieldsInput<F extends Record<string, ConfigField>> = {
     [K in keyof F]?: z.output<F[K]['schema']>;
@@ -101,9 +93,6 @@ export interface Configuration extends ResolvedConfigValues {
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
  * `headless` | `CRAWLEE_HEADLESS` | `true`
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
  * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
@@ -114,7 +103,6 @@ export interface Configuration extends ResolvedConfigValues {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
@@ -123,12 +111,12 @@ export interface Configuration extends ResolvedConfigValues {
  * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
 export declare class Configuration {
+    #private;
     /**
      * Field definitions for this configuration class.
      * Subclasses override this to register additional fields.
      */
     protected static fields: Record<string, ConfigField>;
-    private resolvedValues;
     /**
      * Creates new `Configuration` instance with provided options.
      * Constructor options take precedence over environment variables, which take precedence
@@ -141,23 +129,4 @@ export declare class Configuration {
      * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
     static getGlobalConfiguration(): Configuration;
-    /**
-     * Resolves all field values once using the priority chain:
-     * constructor options > env vars > crawlee.json > schema defaults.
-     */
-    private static resolveAll;
-    /**
-     * Registers getters (and throwing setters) on the instance for each field.
-     */
-    private registerAccessors;
-    /**
-     * Reads the first defined env var value for a field definition.
-     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
-     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
-     */
-    private static readEnvVar;
-    /**
-     * Loads config options from crawlee.json in the current working directory.
-     */
-    private static loadFileOptions;
 }

package/configuration.js

@@ -36,17 +36,11 @@ const logLevelSchema = z.preprocess((val) => {
     if (key in LogLevel)
         return LogLevel[key];
     return val;
-}, z.nativeEnum(LogLevel));
+}, z.enum(LogLevel));
 // --- Crawlee config field definitions ---
 export const crawleeConfigFields = {
-    /** @default 'default' */
-    defaultDatasetId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_DATASET_ID'),
     /** @default true */
     purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
-    /** @default 'default' */
-    defaultKeyValueStoreId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID'),
-    /** @default 'default' */
-    defaultRequestQueueId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID'),
     /** @default 0.95 */
     maxUsedCpuRatio: field(coerceNumber.default(0.95)),
     /** @default 0.25 */
@@ -61,8 +55,6 @@ export const crawleeConfigFields = {
     internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
     /** @default 1_000 */
     systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
-    /** @default 'INPUT' */
-    inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
     /** @default true */
     headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
     /** @default false */
@@ -119,9 +111,6 @@ export const crawleeConfigFields = {
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
  * `headless` | `CRAWLEE_HEADLESS` | `true`
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
  * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
@@ -132,7 +121,6 @@ export const crawleeConfigFields = {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
@@ -147,7 +135,7 @@ export class Configuration {
      * Subclasses override this to register additional fields.
      */
     static fields = crawleeConfigFields;
-    resolvedValues;
+    #resolvedValues;
     /**
      * Creates new `Configuration` instance with provided options.
      * Constructor options take precedence over environment variables, which take precedence
@@ -155,9 +143,9 @@ export class Configuration {
      */
     constructor(options = {}) {
         const fields = this.constructor.fields;
-        const fileOptions = Configuration.loadFileOptions();
-        this.resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
-        this.registerAccessors();
+        const fileOptions = Configuration.#loadFileOptions();
+        this.#resolvedValues = Configuration.#resolveAll(fields, options, fileOptions);
+        this.#registerAccessors();
         // Set the log level
         const logLevel = this.logLevel;
         if (logLevel != null) {
@@ -176,7 +164,7 @@ export class Configuration {
      * Resolves all field values once using the priority chain:
      * constructor options > env vars > crawlee.json > schema defaults.
      */
-    static resolveAll(fields, userOptions, fileOptions) {
+    static #resolveAll(fields, userOptions, fileOptions) {
         const values = {};
         for (const [key, fieldDef] of Object.entries(fields)) {
             // 1. Constructor options (highest priority)
@@ -185,7 +173,7 @@ export class Configuration {
                 continue;
             }
             // 2. Environment variables
-            const envValue = Configuration.readEnvVar(fieldDef);
+            const envValue = Configuration.#readEnvVar(fieldDef);
             if (envValue != null) {
                 values[key] = fieldDef.schema.parse(envValue);
                 continue;
@@ -196,20 +184,20 @@ export class Configuration {
                 continue;
             }
             // 4. Schema default (by parsing undefined through the schema)
-            const result = fieldDef.schema.safeParse(undefined);
-            values[key] = result.success ? result.data : undefined;
+            const parsed = fieldDef.schema.safeParse(undefined);
+            values[key] = parsed.success ? parsed.data : undefined;
         }
         return values;
     }
     /**
      * Registers getters (and throwing setters) on the instance for each field.
      */
-    registerAccessors() {
+    #registerAccessors() {
         const fields = this.constructor.fields;
         const descriptors = {};
         for (const key of Object.keys(fields)) {
             descriptors[key] = {
-                get: () => this.resolvedValues[key],
+                get: () => this.#resolvedValues[key],
                 set() {
                     throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
                 },
@@ -224,7 +212,7 @@ export class Configuration {
      * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
      * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static readEnvVar(fieldDef) {
+    static #readEnvVar(fieldDef) {
         if (!fieldDef.envVar)
             return undefined;
         const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
@@ -238,7 +226,7 @@ export class Configuration {
     /**
      * Loads config options from crawlee.json in the current working directory.
      */
-    static loadFileOptions() {
+    static #loadFileOptions() {
         try {
             const file = readFileSync(join(process.cwd(), 'crawlee.json'));
             return JSON.parse(file.toString());

package/debug.js

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import ow from 'ow';
+import { parseArgument, schemas } from './validators.js';
 /**
  * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
  *
@@ -12,9 +12,9 @@ import ow from 'ow';
  * @internal
  */
 export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
-    ow(request, ow.object);
-    ow(response, ow.object);
-    ow(additionalFields, ow.object);
+    parseArgument(request, schemas.anyObject);
+    parseArgument(response, schemas.anyObject);
+    parseArgument(additionalFields, schemas.anyObject);
     return {
         requestId: request.id,
         url: request.url,

package/errors.d.ts

@@ -10,9 +10,14 @@ export declare class NonRetryableError extends Error {
 export declare class CriticalError extends NonRetryableError {
 }
 /**
- * @ignore
+ * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
+ * error types do not have to depend on `@standard-schema/spec`.
  */
-export declare class MissingRouteError extends CriticalError {
+export interface SchemaIssue {
+    readonly message: string;
+    readonly path?: readonly (PropertyKey | {
+        key: PropertyKey;
+    })[];
 }
 /**
  * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
@@ -21,26 +26,19 @@ export declare class MissingRouteError extends CriticalError {
  */
 export declare class RequestValidationError extends NonRetryableError {
     readonly label: string | symbol;
-    readonly issues: readonly {
-        readonly message: string;
-        readonly path?: readonly (PropertyKey | {
-            key: PropertyKey;
-        })[];
-    }[];
-    constructor(label: string | symbol, issues: readonly {
-        readonly message: string;
-        readonly path?: readonly (PropertyKey | {
-            key: PropertyKey;
-        })[];
-    }[]);
+    readonly issues: readonly SchemaIssue[];
+    constructor(label: string | symbol, issues: readonly SchemaIssue[]);
 }
 /**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
  *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
  */
-export declare class RetryRequestError extends Error {
-    constructor(message?: string);
+export declare class StateValidationError extends Error {
+    readonly persistStateKey: string;
+    readonly issues: readonly SchemaIssue[];
+    constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
 }
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
@@ -51,22 +49,20 @@ export declare class SessionError extends Error {
     constructor(message?: string);
 }
 /**
- * Thrown when a requested session is not found in the referenced SessionPool.
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
  */
-export declare class MissingSessionError extends Error {
-    constructor(sessionId?: string);
-}
-export declare class ContextPipelineInterruptedError extends Error {
-    constructor(message?: string);
+export declare class SessionRetiredError extends SessionError {
 }
-export declare class ContextPipelineInitializationError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
-export declare class ContextPipelineCleanupError extends CriticalError {
-    constructor(error: unknown, options?: ErrorOptions);
-}
-export declare class RequestHandlerError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export declare class AfterCommitError extends NonRetryableError {
+    constructor(cause: unknown);
 }
 /**
  * Thrown when attempting to set a different service instance after one has already been retrieved.
@@ -74,9 +70,3 @@ export declare class RequestHandlerError extends Error {
 export declare class ServiceConflictError extends Error {
     constructor(serviceName: string, newValue: unknown, existingValue: unknown);
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export declare class NavigationSkippedError extends NonRetryableError {
-}

package/errors.js

@@ -10,10 +10,15 @@ export class NonRetryableError extends Error {
  */
 export class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export class MissingRouteError extends CriticalError {
+function formatIssues(issues) {
+    return issues
+        .map((issue) => {
+        const path = (issue.path ?? [])
+            .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+            .join('.');
+        return `- ${path ? `${path}: ` : ''}${issue.message}`;
+    })
+        .join('\n');
 }
 /**
  * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
@@ -24,27 +29,24 @@ export class RequestValidationError extends NonRetryableError {
     label;
     issues;
     constructor(label, issues) {
-        const details = issues
-            .map((issue) => {
-            const path = (issue.path ?? [])
-                .map((segment) => (typeof segment === 'object' ? segment.key : segment))
-                .join('.');
-            return `- ${path ? `${path}: ` : ''}${issue.message}`;
-        })
-            .join('\n');
-        super(`Request userData for label '${String(label)}' failed schema validation:\n${details}`);
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${formatIssues(issues)}`);
         this.label = label;
         this.issues = issues;
     }
 }
 /**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
  *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
  */
-export class RetryRequestError extends Error {
-    constructor(message) {
-        super(message ?? "Request is being retried at the user's request");
+export class StateValidationError extends Error {
+    persistStateKey;
+    issues;
+    constructor(persistStateKey, issues) {
+        super(`State persisted under key '${persistStateKey}' failed schema validation:\n${formatIssues(issues)}`);
+        this.persistStateKey = persistStateKey;
+        this.issues = issues;
     }
 }
 /**
@@ -58,31 +60,21 @@ export class SessionError extends Error {
     }
 }
 /**
- * Thrown when a requested session is not found in the referenced SessionPool.
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
  */
-export class MissingSessionError extends Error {
-    constructor(sessionId) {
-        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
-    }
+export class SessionRetiredError extends SessionError {
 }
-export class ContextPipelineInterruptedError extends Error {
-    constructor(message) {
-        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
-    }
-}
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
-export class ContextPipelineCleanupError extends CriticalError {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
-export class RequestHandlerError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export class AfterCommitError extends NonRetryableError {
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
     }
 }
 /**
@@ -94,9 +86,3 @@ export class ServiceConflictError extends Error {
             `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
     }
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export class NavigationSkippedError extends NonRetryableError {
-}

package/events/event_manager.d.ts

@@ -4,7 +4,7 @@ export interface EventManagerOptions {
     /** Interval between emitted `persistState` events in milliseconds. */
     persistStateIntervalMillis: number;
 }
-export declare const enum EventType {
+export declare enum EventType {
     PERSIST_STATE = "persistState",
     SYSTEM_INFO = "systemInfo",
     MIGRATING = "migrating",
@@ -41,12 +41,12 @@ interface Intervals {
     systemInfo?: BetterIntervalID;
 }
 export declare abstract class EventManager {
+    #private;
     protected events: AsyncEventEmitter<{}>;
     protected initialized: boolean;
     protected intervals: Intervals;
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected log: import("@crawlee/types").CrawleeLogger;
-    private persistStateIntervalMillis;
     constructor(options: EventManagerOptions);
     /**
      * Initializes the event manager by starting the `persistState` event interval.

package/events/event_manager.js

@@ -15,10 +15,11 @@ export class EventManager {
     initialized = false;
     intervals = {};
     log = serviceLocator.getLogger().child({ prefix: 'Events' });
-    persistStateIntervalMillis;
+    #persistStateIntervalMillis;
     constructor(options) {
-        this.persistStateIntervalMillis = options.persistStateIntervalMillis;
-        this.events.setMaxListeners(50);
+        this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
+        // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
+        this.events.setMaxListeners(150);
     }
     /**
      * Initializes the event manager by starting the `persistState` event interval.
@@ -29,9 +30,9 @@ export class EventManager {
             return;
         }
         this.intervals.persistState = betterSetInterval((intervalCallback) => {
-            this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+            this.emit(EventType.PERSIST_STATE, { isMigrating: false });
             intervalCallback();
-        }, this.persistStateIntervalMillis);
+        }, this.#persistStateIntervalMillis);
         this.initialized = true;
     }
     /**
@@ -45,7 +46,7 @@ export class EventManager {
         betterClearInterval(this.intervals.persistState);
         this.initialized = false;
         // Emit final PERSIST_STATE event
-        this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+        this.emit(EventType.PERSIST_STATE, { isMigrating: false });
         // Wait for PERSIST_STATE to process
         await this.waitForAllListenersToComplete();
     }

package/events/index.d.ts

@@ -1,2 +1,3 @@
 export * from './event_manager.js';
 export * from './local_event_manager.js';
+export type * from './system_info.js';

package/events/local_event_manager.d.ts

@@ -5,7 +5,7 @@ export interface LocalEventManagerOptions extends EventManagerOptions {
     systemInfoIntervalMillis: number;
 }
 export declare class LocalEventManager extends EventManager {
-    private systemInfoIntervalMillis;
+    #private;
     constructor(options: LocalEventManagerOptions);
     /**
      * Creates a new `LocalEventManager` based on the provided `Configuration`.
@@ -29,11 +29,4 @@ export declare class LocalEventManager extends EventManager {
      * @internal
      */
     isContainerizedWrapper(): Promise<boolean>;
-    /**
-     * Creates a SystemInfo object based on local metrics.
-     */
-    private createSystemInfo;
-    private createCpuInfo;
-    private createMemoryInfo;
-    private getMemoryInfo;
 }

package/events/local_event_manager.js

@@ -1,11 +1,11 @@
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { serviceLocator } from '../service_locator.js';
-import { EventManager } from './event_manager.js';
+import { EventManager, EventType } from './event_manager.js';
 export class LocalEventManager extends EventManager {
-    systemInfoIntervalMillis;
+    #systemInfoIntervalMillis;
     constructor(options) {
         super(options);
-        this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+        this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;
     }
     /**
      * Creates a new `LocalEventManager` based on the provided `Configuration`.
@@ -28,7 +28,7 @@ export class LocalEventManager extends EventManager {
         }
         await super.init();
         this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
-        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.#systemInfoIntervalMillis);
     }
     /**
      * @inheritDoc
@@ -44,10 +44,10 @@ export class LocalEventManager extends EventManager {
      * @internal
      */
     async emitSystemInfoEvent(intervalCallback) {
-        const info = await this.createSystemInfo({
+        const info = await this.#createSystemInfo({
             maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
         });
-        this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
+        this.events.emit(EventType.SYSTEM_INFO, info);
         intervalCallback();
     }
     /**
@@ -60,14 +60,14 @@ export class LocalEventManager extends EventManager {
     /**
      * Creates a SystemInfo object based on local metrics.
      */
-    async createSystemInfo(options) {
+    async #createSystemInfo(options) {
         return {
             createdAt: new Date(),
-            ...(await this.createCpuInfo(options)),
-            ...(await this.createMemoryInfo()),
+            ...(await this.#createCpuInfo(options)),
+            ...(await this.#createMemoryInfo()),
         };
     }
-    async createCpuInfo(options) {
+    async #createCpuInfo(options) {
         const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
         const usedCpuRatio = await getCurrentCpuTicksV2({
             containerized: await this.isContainerizedWrapper(),
@@ -78,9 +78,9 @@ export class LocalEventManager extends EventManager {
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
         };
     }
-    async createMemoryInfo() {
+    async #createMemoryInfo() {
         try {
-            const memInfo = await this.getMemoryInfo();
+            const memInfo = await this.#getMemoryInfo();
             return {
                 memTotalBytes: memInfo.totalBytes,
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
@@ -91,7 +91,7 @@ export class LocalEventManager extends EventManager {
             return {};
         }
     }
-    async getMemoryInfo() {
+    async #getMemoryInfo() {
         const { getMemoryInfo } = await import('../system-info/memory-info.js');
         return getMemoryInfo({
             containerized: await this.isContainerizedWrapper(),