@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/configuration.d.ts

@@ -6,47 +6,39 @@ export interface ConfigField<T extends z.ZodType = z.ZodType> {
 }
 export declare function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;
 /** Zod preprocessor treating `'0'` and `'false'` as falsy. */
-export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean>;
-export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber>;
+export declare const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;
+export declare const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;
 export declare const crawleeConfigFields: {
-    /** @default 'default' */
-    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
     /** @default true */
-    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
-    /** @default 'default' */
-    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
-    /** @default 'default' */
-    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
+    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default 0.95 */
-    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
+    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 0.25 */
-    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
-    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
+    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
+    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 60_000 */
-    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
+    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /**
      * Internal safety-net timeout for a single request, in milliseconds. When unset the crawler derives it from
      * the request handler timeout (twice it, and never below 5 minutes).
      */
-    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
+    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default 1_000 */
-    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
-    /** @default 'INPUT' */
-    inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
+    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
     /** @default true */
-    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
+    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default false */
-    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
+    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
     defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
     /** @default false */
-    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
-    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>>>>;
+    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
+    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
     /** @default true */
-    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
+    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
     /** @default './storage' */
     storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
-    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
+    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
 };
 export type FieldsInput<F extends Record<string, ConfigField>> = {
     [K in keyof F]?: z.output<F[K]['schema']>;
@@ -101,9 +93,6 @@ export interface Configuration extends ResolvedConfigValues {
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
  * `headless` | `CRAWLEE_HEADLESS` | `true`
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
  * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
@@ -114,7 +103,6 @@ export interface Configuration extends ResolvedConfigValues {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
@@ -141,23 +129,4 @@ export declare class Configuration {
      * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
     static getGlobalConfiguration(): Configuration;
-    /**
-     * Resolves all field values once using the priority chain:
-     * constructor options > env vars > crawlee.json > schema defaults.
-     */
-    private static resolveAll;
-    /**
-     * Registers getters (and throwing setters) on the instance for each field.
-     */
-    private registerAccessors;
-    /**
-     * Reads the first defined env var value for a field definition.
-     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
-     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
-     */
-    private static readEnvVar;
-    /**
-     * Loads config options from crawlee.json in the current working directory.
-     */
-    private static loadFileOptions;
 }

package/configuration.js

@@ -39,14 +39,8 @@ const logLevelSchema = z.preprocess((val) => {
 }, z.enum(LogLevel));
 // --- Crawlee config field definitions ---
 export const crawleeConfigFields = {
-    /** @default 'default' */
-    defaultDatasetId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_DATASET_ID'),
     /** @default true */
     purgeOnStart: field(coerceBoolean.default(true), 'CRAWLEE_PURGE_ON_START'),
-    /** @default 'default' */
-    defaultKeyValueStoreId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID'),
-    /** @default 'default' */
-    defaultRequestQueueId: field(z.string().default('default'), 'CRAWLEE_DEFAULT_REQUEST_QUEUE_ID'),
     /** @default 0.95 */
     maxUsedCpuRatio: field(coerceNumber.default(0.95)),
     /** @default 0.25 */
@@ -61,8 +55,6 @@ export const crawleeConfigFields = {
     internalTimeoutMillis: field(coerceNumber.optional(), 'CRAWLEE_INTERNAL_TIMEOUT'),
     /** @default 1_000 */
     systemInfoIntervalMillis: field(coerceNumber.default(1_000)),
-    /** @default 'INPUT' */
-    inputKey: field(z.string().default('INPUT'), 'CRAWLEE_INPUT_KEY'),
     /** @default true */
     headless: field(coerceBoolean.default(true), 'CRAWLEE_HEADLESS'),
     /** @default false */
@@ -119,9 +111,6 @@ export const crawleeConfigFields = {
  * `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
  * `logLevel` | `CRAWLEE_LOG_LEVEL` | -
  * `headless` | `CRAWLEE_HEADLESS` | `true`
- * `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
- * `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
- * `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
  * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
  * `internalTimeoutMillis` | `CRAWLEE_INTERNAL_TIMEOUT` | -
  * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
@@ -132,7 +121,6 @@ export const crawleeConfigFields = {
  *
  * Key | Environment Variable | Default Value
  * ---|---|---
- * `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
  * `xvfb` | `CRAWLEE_XVFB` | `false`
  * `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
  * `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
@@ -155,9 +143,9 @@ export class Configuration {
      */
     constructor(options = {}) {
         const fields = this.constructor.fields;
-        const fileOptions = Configuration.loadFileOptions();
-        this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
-        this.registerAccessors();
+        const fileOptions = Configuration.#loadFileOptions();
+        this.#resolvedValues = Configuration.#resolveAll(fields, options, fileOptions);
+        this.#registerAccessors();
         // Set the log level
         const logLevel = this.logLevel;
         if (logLevel != null) {
@@ -176,7 +164,7 @@ export class Configuration {
      * Resolves all field values once using the priority chain:
      * constructor options > env vars > crawlee.json > schema defaults.
      */
-    static resolveAll(fields, userOptions, fileOptions) {
+    static #resolveAll(fields, userOptions, fileOptions) {
         const values = {};
         for (const [key, fieldDef] of Object.entries(fields)) {
             // 1. Constructor options (highest priority)
@@ -185,7 +173,7 @@ export class Configuration {
                 continue;
             }
             // 2. Environment variables
-            const envValue = Configuration.readEnvVar(fieldDef);
+            const envValue = Configuration.#readEnvVar(fieldDef);
             if (envValue != null) {
                 values[key] = fieldDef.schema.parse(envValue);
                 continue;
@@ -204,7 +192,7 @@ export class Configuration {
     /**
      * Registers getters (and throwing setters) on the instance for each field.
      */
-    registerAccessors() {
+    #registerAccessors() {
         const fields = this.constructor.fields;
         const descriptors = {};
         for (const key of Object.keys(fields)) {
@@ -224,7 +212,7 @@ export class Configuration {
      * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
      * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static readEnvVar(fieldDef) {
+    static #readEnvVar(fieldDef) {
         if (!fieldDef.envVar)
             return undefined;
         const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
@@ -238,7 +226,7 @@ export class Configuration {
     /**
      * Loads config options from crawlee.json in the current working directory.
      */
-    static loadFileOptions() {
+    static #loadFileOptions() {
         try {
             const file = readFileSync(join(process.cwd(), 'crawlee.json'));
             return JSON.parse(file.toString());

package/errors.d.ts

@@ -9,11 +9,6 @@ export declare class NonRetryableError extends Error {
  */
 export declare class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export declare class MissingRouteError extends CriticalError {
-}
 /**
  * A schema validation issue, structurally compatible with `StandardSchemaV1.Issue`. Declared here so that
  * error types do not have to depend on `@standard-schema/spec`.
@@ -45,34 +40,6 @@ export declare class StateValidationError extends Error {
     readonly issues: readonly SchemaIssue[];
     constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
 }
-/**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
- *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
- */
-export declare class RetryRequestError extends Error {
-    constructor(message?: string);
-}
-/**
- * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
- *
- * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
- * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
- * the domain's backoff expires, so retries are paced rather than immediate.
- */
-export declare class RequestThrottledError extends RetryRequestError {
-    constructor(message?: string);
-}
-/**
- * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
- * abandoned rather than kept waiting.
- *
- * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
- * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
- * resumes them once the domain recovers.
- */
-export declare class PersistentRateLimitError extends CriticalError {
-}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
@@ -82,22 +49,20 @@ export declare class SessionError extends Error {
     constructor(message?: string);
 }
 /**
- * Thrown when a requested session is not found in the referenced SessionPool.
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
  */
-export declare class MissingSessionError extends Error {
-    constructor(sessionId?: string);
-}
-export declare class ContextPipelineInterruptedError extends Error {
-    constructor(message?: string);
-}
-export declare class ContextPipelineInitializationError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
-}
-export declare class ContextPipelineCleanupError extends CriticalError {
-    constructor(error: unknown, options?: ErrorOptions);
+export declare class SessionRetiredError extends SessionError {
 }
-export declare class RequestHandlerError extends Error {
-    constructor(error: unknown, options?: ErrorOptions);
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export declare class AfterCommitError extends NonRetryableError {
+    constructor(cause: unknown);
 }
 /**
  * Thrown when attempting to set a different service instance after one has already been retrieved.
@@ -105,9 +70,3 @@ export declare class RequestHandlerError extends Error {
 export declare class ServiceConflictError extends Error {
     constructor(serviceName: string, newValue: unknown, existingValue: unknown);
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export declare class NavigationSkippedError extends NonRetryableError {
-}

package/errors.js

@@ -10,11 +10,6 @@ export class NonRetryableError extends Error {
  */
 export class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export class MissingRouteError extends CriticalError {
-}
 function formatIssues(issues) {
     return issues
         .map((issue) => {
@@ -54,38 +49,6 @@ export class StateValidationError extends Error {
         this.issues = issues;
     }
 }
-/**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
- *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
- */
-export class RetryRequestError extends Error {
-    constructor(message) {
-        super(message ?? "Request is being retried at the user's request");
-    }
-}
-/**
- * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
- *
- * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
- * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
- * the domain's backoff expires, so retries are paced rather than immediate.
- */
-export class RequestThrottledError extends RetryRequestError {
-    constructor(message) {
-        super(message ?? 'Request is being retried later because its domain is rate-limiting us');
-    }
-}
-/**
- * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
- * abandoned rather than kept waiting.
- *
- * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
- * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
- * resumes them once the domain recovers.
- */
-export class PersistentRateLimitError extends CriticalError {
-}
 /**
  * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
  *
@@ -97,31 +60,21 @@ export class SessionError extends Error {
     }
 }
 /**
- * Thrown when a requested session is not found in the referenced SessionPool.
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
  */
-export class MissingSessionError extends Error {
-    constructor(sessionId) {
-        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
-    }
-}
-export class ContextPipelineInterruptedError extends Error {
-    constructor(message) {
-        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
-    }
-}
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
+export class SessionRetiredError extends SessionError {
 }
-export class ContextPipelineCleanupError extends CriticalError {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
-}
-export class RequestHandlerError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export class AfterCommitError extends NonRetryableError {
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
     }
 }
 /**
@@ -133,9 +86,3 @@ export class ServiceConflictError extends Error {
             `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
     }
 }
-/**
- * Thrown by crawlers when `skipNavigation` is used on a request.
- * Subclasses can catch this error to skip their own navigation-dependent logic.
- */
-export class NavigationSkippedError extends NonRetryableError {
-}

package/events/index.d.ts

@@ -1,2 +1,3 @@
 export * from './event_manager.js';
 export * from './local_event_manager.js';
+export type * from './system_info.js';

package/events/local_event_manager.d.ts

@@ -29,11 +29,4 @@ export declare class LocalEventManager extends EventManager {
      * @internal
      */
     isContainerizedWrapper(): Promise<boolean>;
-    /**
-     * Creates a SystemInfo object based on local metrics.
-     */
-    private createSystemInfo;
-    private createCpuInfo;
-    private createMemoryInfo;
-    private getMemoryInfo;
 }

package/events/local_event_manager.js

@@ -44,7 +44,7 @@ export class LocalEventManager extends EventManager {
      * @internal
      */
     async emitSystemInfoEvent(intervalCallback) {
-        const info = await this.createSystemInfo({
+        const info = await this.#createSystemInfo({
             maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
         });
         this.events.emit(EventType.SYSTEM_INFO, info);
@@ -60,14 +60,14 @@ export class LocalEventManager extends EventManager {
     /**
      * Creates a SystemInfo object based on local metrics.
      */
-    async createSystemInfo(options) {
+    async #createSystemInfo(options) {
         return {
             createdAt: new Date(),
-            ...(await this.createCpuInfo(options)),
-            ...(await this.createMemoryInfo()),
+            ...(await this.#createCpuInfo(options)),
+            ...(await this.#createMemoryInfo()),
         };
     }
-    async createCpuInfo(options) {
+    async #createCpuInfo(options) {
         const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
         const usedCpuRatio = await getCurrentCpuTicksV2({
             containerized: await this.isContainerizedWrapper(),
@@ -78,9 +78,9 @@ export class LocalEventManager extends EventManager {
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
         };
     }
-    async createMemoryInfo() {
+    async #createMemoryInfo() {
         try {
-            const memInfo = await this.getMemoryInfo();
+            const memInfo = await this.#getMemoryInfo();
             return {
                 memTotalBytes: memInfo.totalBytes,
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
@@ -91,7 +91,7 @@ export class LocalEventManager extends EventManager {
             return {};
         }
     }
-    async getMemoryInfo() {
+    async #getMemoryInfo() {
         const { getMemoryInfo } = await import('../system-info/memory-info.js');
         return getMemoryInfo({
             containerized: await this.isContainerizedWrapper(),

package/events/system_info.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Represents the current status of the system.
+ */
+export interface SystemInfo {
+    /** If false, system is being overloaded. */
+    isSystemIdle: boolean;
+    memInfo: LoadSignalInfo;
+    eventLoopInfo: LoadSignalInfo;
+    cpuInfo: LoadSignalInfo;
+    storageBackendInfo: LoadSignalInfo;
+    memTotalBytes?: number;
+    memCurrentBytes?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    cpuCurrentUsage?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    isCpuOverloaded?: boolean;
+    /**
+     * Platform only property
+     * @internal
+     */
+    createdAt?: Date;
+    /**
+     * Status of additional load signals beyond the built-in four.
+     * Keys are `LoadSignal.name` values, values are overload info.
+     */
+    loadSignalInfo?: Record<string, LoadSignalInfo>;
+}
+export interface LoadSignalInfo {
+    isOverloaded: boolean;
+    limitRatio: number;
+    actualRatio: number;
+}

package/index.d.ts

@@ -1,22 +1,16 @@
 export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
 export * from './service_locator.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
 export * from './events/index.js';
 export * from './log.js';
 export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
-export * from './validators.js';
-export * from './cookie_utils.js';
-export * from './http.js';
+export { ArgumentValidationError, validators } from './validators.js';
 export * from './recoverable_state.js';
 export type { StorageBackend } from '@crawlee/types';
+export { EnqueueStrategy } from '@crawlee/utils';

package/index.js

@@ -1,21 +1,17 @@
 export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
 export * from './service_locator.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
 export * from './events/index.js';
 export * from './log.js';
 export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
 export * from './memory-storage/index.js';
-export * from './validators.js';
-export * from './cookie_utils.js';
-export * from './http.js';
+// Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no
+// semver guarantees and must not reach the public surface. Internal consumers import them directly.
+export { ArgumentValidationError, validators } from './validators.js';
 export * from './recoverable_state.js';
+export { EnqueueStrategy } from '@crawlee/utils';

package/internal.d.ts

@@ -0,0 +1,8 @@
+export * from './iterables.js';
+export * from './storages/batched_adds.js';
+export { joinRequestSourceStatuses } from './storages/request_loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/internal.js

@@ -0,0 +1,9 @@
+// Not part of the public API: internals that other Crawlee packages need but that carry no semver guarantees.
+export * from './iterables.js';
+export * from './storages/batched_adds.js';
+export { joinRequestSourceStatuses } from './storages/request_loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/log.d.ts

@@ -1,7 +1,7 @@
-import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+import type { CrawleeLogger, CrawleeLoggerOptions, LogOptions } from '@crawlee/types';
 import type { LoggerOptions } from '@apify/log';
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
-export type { CrawleeLogger, CrawleeLoggerOptions };
+export type { CrawleeLogger, CrawleeLoggerOptions, LogOptions };
 /**
  * Abstract base class for custom Crawlee logger implementations.
  *
@@ -33,8 +33,7 @@ export type { CrawleeLogger, CrawleeLoggerOptions };
  * ```
  */
 export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
-    private options;
-    private readonly warningsLogged;
+    #private;
     constructor(options?: Partial<CrawleeLoggerOptions>);
     /**
      * Core logging method. Subclasses must implement this to dispatch log messages
@@ -56,14 +55,14 @@ export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
     getOptions(): CrawleeLoggerOptions;
     setOptions(options: Partial<CrawleeLoggerOptions>): void;
     child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
-    error(message: string, data?: Record<string, unknown>): void;
+    error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     exception(exception: Error, message: string, data?: Record<string, unknown>): void;
-    softFail(message: string, data?: Record<string, unknown>): void;
-    warning(message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
+    warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     warningOnce(message: string): void;
-    info(message: string, data?: Record<string, unknown>): void;
-    debug(message: string, data?: Record<string, unknown>): void;
-    perf(message: string, data?: Record<string, unknown>): void;
+    info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
+    debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
+    perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
     deprecated(message: string): void;
 }
 /**
@@ -73,7 +72,7 @@ export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
  * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
  */
 export declare class ApifyLogAdapter extends BaseCrawleeLogger {
-    private readonly apifyLog;
+    #private;
     constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;