@crawlee/core 4.0.0-beta.25 → 4.0.0-beta.251

package/errors.js

@@ -1,3 +1,4 @@
+import { inspectValue } from './debug.js';
 /**
  * Errors of `NonRetryableError` type will never be retried by the crawler.
  */
@@ -9,49 +10,79 @@ export class NonRetryableError extends Error {
  */
 export class CriticalError extends NonRetryableError {
 }
-/**
- * @ignore
- */
-export class MissingRouteError extends CriticalError {
+function formatIssues(issues) {
+    return issues
+        .map((issue) => {
+        const path = (issue.path ?? [])
+            .map((segment) => (typeof segment === 'object' ? segment.key : segment))
+            .join('.');
+        return `- ${path ? `${path}: ` : ''}${issue.message}`;
+    })
+        .join('\n');
 }
 /**
- * Errors of `RetryRequestError` type will always be retried by the crawler.
+ * Thrown when a request's `userData` does not match the {@link RouteSchemas|Standard Schema} registered for its label.
  *
- * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ * As the `userData` does not change between attempts, this error is non-retryable.
  */
-export class RetryRequestError extends Error {
-    constructor(message) {
-        super(message ?? "Request is being retried at the user's request");
+export class RequestValidationError extends NonRetryableError {
+    label;
+    issues;
+    constructor(label, issues) {
+        super(`Request userData for label '${String(label)}' failed schema validation:\n${formatIssues(issues)}`);
+        this.label = label;
+        this.issues = issues;
     }
 }
 /**
- * Errors of `SessionError` type will trigger a session rotation.
+ * Thrown by {@link RecoverableState} when a persisted state record does not match its `stateSchema`.
  *
- * This error doesn't respect the `maxRequestRetries` option and has a separate limit of `maxSessionRotations`.
+ * Whether a corrupt record should abort the run or be discarded in favour of the defaults depends on what the
+ * state is for, so {@link RecoverableState.initialize} always throws and leaves the choice to the caller.
  */
-export class SessionError extends RetryRequestError {
-    constructor(message) {
-        super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
+export class StateValidationError extends Error {
+    persistStateKey;
+    issues;
+    constructor(persistStateKey, issues) {
+        super(`State persisted under key '${persistStateKey}' failed schema validation:\n${formatIssues(issues)}`);
+        this.persistStateKey = persistStateKey;
+        this.issues = issues;
     }
 }
-export class ContextPipelineInterruptedError extends Error {
+/**
+ * Errors of `SessionError` type retire the session associated with the request and trigger a regular retry.
+ *
+ * The retry counts towards the `maxRequestRetries` limit, just like any other error.
+ */
+export class SessionError extends Error {
     constructor(message) {
-        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+        super(`Detected a session error, retiring session... ${message ? `\n${message}` : ''}`);
     }
 }
-export class ContextPipelineInitializationError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
-    }
+/**
+ * A {@link SessionError} for a session that finished without being blocked, e.g. by reaching its
+ * `maxUsageCount` or `maxAgeSecs`. A plain `SessionError` means the session was blocked.
+ */
+export class SessionRetiredError extends SessionError {
 }
-export class ContextPipelineCleanupError extends CriticalError {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
+/**
+ * Wraps the failure of a callback registered with {@link StorageTransaction.afterCommit|`afterCommit`}
+ * that ran after the request's writes had already been committed.
+ *
+ * Non-retryable by nature: the writes are durable, so re-running the request handler would duplicate
+ * them. A callback that throws a `NonRetryableError` of its own is left alone.
+ */
+export class AfterCommitError extends NonRetryableError {
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
     }
 }
-export class RequestHandlerError extends Error {
-    constructor(error, options) {
-        super(undefined, { cause: error, ...options });
+/**
+ * Thrown when attempting to set a different service instance after one has already been retrieved.
+ */
+export class ServiceConflictError extends Error {
+    constructor(serviceName, newValue, existingValue) {
+        super(`Service ${serviceName} is already in use. ` +
+            `Existing value: ${inspectValue(existingValue)}, attempted new value: ${inspectValue(newValue)}.`);
     }
 }
-//# sourceMappingURL=errors.js.map

package/events/event-manager.d.ts

@@ -0,0 +1,78 @@
+import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
+import type { BetterIntervalID } from '@apify/utilities';
+export interface EventManagerOptions {
+    /** Interval between emitted `persistState` events in milliseconds. */
+    persistStateIntervalMillis: number;
+}
+export declare enum EventType {
+    PERSIST_STATE = "persistState",
+    SYSTEM_INFO = "systemInfo",
+    MIGRATING = "migrating",
+    ABORTING = "aborting",
+    EXIT = "exit",
+    STATUS_MESSAGE = "statusMessage"
+}
+export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';
+/**
+ * Payload emitted with the {@link EventType.STATUS_MESSAGE|`statusMessage`} event.
+ *
+ * The crawler broadcasts these whenever it wants to report its progress (e.g. periodically, or on
+ * start/finish). Consumers such as the Apify SDK can listen for the event and propagate the message
+ * to the platform. This keeps the crawler decoupled from any specific status-reporting backend.
+ */
+export interface EventStatusMessageData {
+    /**
+     * Identifies the crawler that emitted the message.
+     *
+     * Either the user-provided `id` from the crawler options, or a randomly generated one.
+     * Since a single event manager may be shared by multiple crawlers, consumers can use this
+     * to attribute the message to a specific crawler instance.
+     */
+    crawlerId: string;
+    /** The human-readable status message. */
+    message: string;
+    /** Whether this is the final status message of the run. */
+    isStatusMessageTerminal?: boolean;
+    /** The log level the message was logged with. */
+    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
+}
+interface Intervals {
+    persistState?: BetterIntervalID;
+    systemInfo?: BetterIntervalID;
+}
+export declare abstract class EventManager {
+    #private;
+    protected events: AsyncEventEmitter<{}>;
+    protected initialized: boolean;
+    protected intervals: Intervals;
+// @ts-ignore optional peer dependency or compatibility with es2022
+    protected log: import("@crawlee/types").CrawleeLogger;
+    constructor(options: EventManagerOptions);
+    /**
+     * Initializes the event manager by starting the `persistState` event interval.
+     * This is automatically called at the beginning of `crawler.run()`.
+     */
+    init(): Promise<void>;
+    /**
+     * Clears the internal `persistState` event interval.
+     * This is automatically called at the end of `crawler.run()`.
+     */
+    close(): Promise<void>;
+    on(event: EventTypeName, listener: (...args: any[]) => any): void;
+    off(event: EventTypeName, listener?: (...args: any[]) => any): void;
+    emit(event: EventTypeName, ...args: unknown[]): void;
+    isInitialized(): boolean;
+    /**
+     * @internal
+     */
+    listenerCount(event: EventTypeName): number;
+    /**
+     * @internal
+     */
+    listeners(event: EventTypeName): (() => Promise<unknown>)[];
+    /**
+     * @internal
+     */
+    waitForAllListenersToComplete(): Promise<boolean>;
+}
+export {};

package/events/{event_manager.js → event-manager.js}

@@ -1,7 +1,6 @@
 import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
-import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service-locator.js';
 export var EventType;
 (function (EventType) {
     EventType["PERSIST_STATE"] = "persistState";
@@ -9,30 +8,31 @@ export var EventType;
     EventType["MIGRATING"] = "migrating";
     EventType["ABORTING"] = "aborting";
     EventType["EXIT"] = "exit";
+    EventType["STATUS_MESSAGE"] = "statusMessage";
 })(EventType || (EventType = {}));
 export class EventManager {
-    config;
     events = new AsyncEventEmitter();
     initialized = false;
     intervals = {};
-    log = log.child({ prefix: 'Events' });
-    constructor(config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.events.setMaxListeners(50);
+    log = serviceLocator.getLogger().child({ prefix: 'Events' });
+    #persistStateIntervalMillis;
+    constructor(options) {
+        this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
+        // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
+        this.events.setMaxListeners(150);
     }
     /**
-     * Initializes the event manager by creating the `persistState` event interval.
+     * Initializes the event manager by starting the `persistState` event interval.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     async init() {
         if (this.initialized) {
             return;
         }
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
         this.intervals.persistState = betterSetInterval((intervalCallback) => {
-            this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+            this.emit(EventType.PERSIST_STATE, { isMigrating: false });
             intervalCallback();
-        }, persistStateIntervalMillis);
+        }, this.#persistStateIntervalMillis);
         this.initialized = true;
     }
     /**
@@ -46,7 +46,7 @@ export class EventManager {
         betterClearInterval(this.intervals.persistState);
         this.initialized = false;
         // Emit final PERSIST_STATE event
-        this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+        this.emit(EventType.PERSIST_STATE, { isMigrating: false });
         // Wait for PERSIST_STATE to process
         await this.waitForAllListenersToComplete();
     }
@@ -86,4 +86,3 @@ export class EventManager {
         return this.events.waitForAllListenersToComplete();
     }
 }
-//# sourceMappingURL=event_manager.js.map

package/events/index.d.ts

@@ -1,3 +1,3 @@
-export * from './event_manager.js';
-export * from './local_event_manager.js';
-//# sourceMappingURL=index.d.ts.map
+export * from './event-manager.js';
+export * from './local-event-manager.js';
+export type * from './system-info.js';

package/events/index.js

@@ -1,3 +1,2 @@
-export * from './event_manager.js';
-export * from './local_event_manager.js';
-//# sourceMappingURL=index.js.map
+export * from './event-manager.js';
+export * from './local-event-manager.js';

package/events/local-event-manager.d.ts

@@ -0,0 +1,32 @@
+import type { Configuration } from '../configuration.js';
+import { EventManager, type EventManagerOptions } from './event-manager.js';
+export interface LocalEventManagerOptions extends EventManagerOptions {
+    /** Interval between emitted `systemInfo` events in milliseconds. */
+    systemInfoIntervalMillis: number;
+}
+export declare class LocalEventManager extends EventManager {
+    #private;
+    constructor(options: LocalEventManagerOptions);
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfiguration(configuration?: Configuration): LocalEventManager;
+    /**
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
+     * This is automatically called at the beginning of `crawler.run()`.
+     */
+    init(): Promise<void>;
+    /**
+     * @inheritDoc
+     */
+    close(): Promise<void>;
+    /**
+     * @internal
+     */
+    emitSystemInfoEvent(intervalCallback: () => unknown): Promise<void>;
+    /**
+     * @internal
+     */
+    isContainerizedWrapper(): Promise<boolean>;
+}

package/events/local-event-manager.js

@@ -0,0 +1,101 @@
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service-locator.js';
+import { EventManager, EventType } from './event-manager.js';
+export class LocalEventManager extends EventManager {
+    #systemInfoIntervalMillis;
+    constructor(options) {
+        super(options);
+        this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+    }
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfiguration(configuration) {
+        const resolvedConfiguration = configuration ?? serviceLocator.getConfiguration();
+        return new LocalEventManager({
+            persistStateIntervalMillis: resolvedConfiguration.persistStateIntervalMillis,
+            systemInfoIntervalMillis: resolvedConfiguration.systemInfoIntervalMillis,
+        });
+    }
+    /**
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
+     * This is automatically called at the beginning of `crawler.run()`.
+     */
+    async init() {
+        if (this.initialized) {
+            return;
+        }
+        await super.init();
+        this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.#systemInfoIntervalMillis);
+    }
+    /**
+     * @inheritDoc
+     */
+    async close() {
+        if (!this.initialized) {
+            return;
+        }
+        await super.close();
+        betterClearInterval(this.intervals.systemInfo);
+    }
+    /**
+     * @internal
+     */
+    async emitSystemInfoEvent(intervalCallback) {
+        const info = await this.#createSystemInfo({
+            maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
+        });
+        this.events.emit(EventType.SYSTEM_INFO, info);
+        intervalCallback();
+    }
+    /**
+     * @internal
+     */
+    async isContainerizedWrapper() {
+        const { isContainerized } = await import('../system-info/runtime.js');
+        return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
+    }
+    /**
+     * Creates a SystemInfo object based on local metrics.
+     */
+    async #createSystemInfo(options) {
+        return {
+            createdAt: new Date(),
+            ...(await this.#createCpuInfo(options)),
+            ...(await this.#createMemoryInfo()),
+        };
+    }
+    async #createCpuInfo(options) {
+        const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
+        const usedCpuRatio = await getCurrentCpuTicksV2({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
+        return {
+            cpuCurrentUsage: usedCpuRatio * 100,
+            isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
+        };
+    }
+    async #createMemoryInfo() {
+        try {
+            const memInfo = await this.#getMemoryInfo();
+            return {
+                memTotalBytes: memInfo.totalBytes,
+                memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
+            };
+        }
+        catch (err) {
+            this.log.exception(err, 'Memory snapshot failed.');
+            return {};
+        }
+    }
+    async #getMemoryInfo() {
+        const { getMemoryInfo } = await import('../system-info/memory-info.js');
+        return getMemoryInfo({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
+    }
+}

package/events/system-info.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Represents the current status of the system.
+ */
+export interface SystemInfo {
+    /** If false, system is being overloaded. */
+    isSystemIdle: boolean;
+    memInfo: LoadSignalInfo;
+    eventLoopInfo: LoadSignalInfo;
+    cpuInfo: LoadSignalInfo;
+    storageBackendInfo: LoadSignalInfo;
+    memTotalBytes?: number;
+    memCurrentBytes?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    cpuCurrentUsage?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    isCpuOverloaded?: boolean;
+    /**
+     * Platform only property
+     * @internal
+     */
+    createdAt?: Date;
+    /**
+     * Status of additional load signals beyond the built-in four.
+     * Keys are `LoadSignal.name` values, values are overload info.
+     */
+    loadSignalInfo?: Record<string, LoadSignalInfo>;
+}
+export interface LoadSignalInfo {
+    isOverloaded: boolean;
+    limitRatio: number;
+    actualRatio: number;
+}

package/events/system-info.js

@@ -0,0 +1 @@
+export {};

package/index.d.ts

@@ -1,19 +1,16 @@
+export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
+export * from './service-locator.js';
 export * from './events/index.js';
 export * from './log.js';
-export * from './proxy_configuration.js';
+export * from './owned-or-injected.js';
+export * from './proxy-configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
-export * from './validators.js';
-export * from './cookie_utils.js';
-export * from './recoverable_state.js';
-export { PseudoUrl } from '@apify/pseudo_url';
-export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
-//# sourceMappingURL=index.d.ts.map
+export * from './memory-storage/index.js';
+export { ArgumentValidationError, validators } from './validators.js';
+export * from './recoverable-state.js';
+export type { StorageBackend } from '@crawlee/types';
+export { EnqueueStrategy } from '@crawlee/utils';

package/index.js

@@ -1,18 +1,17 @@
+export * from './debug.js';
 export * from './errors.js';
-export * from './autoscaling/index.js';
 export * from './configuration.js';
-export * from './crawlers/index.js';
-export * from './enqueue_links/index.js';
+export * from './service-locator.js';
 export * from './events/index.js';
 export * from './log.js';
-export * from './proxy_configuration.js';
+export * from './owned-or-injected.js';
+export * from './proxy-configuration.js';
 export * from './request.js';
-export * from './router.js';
 export * from './serialization.js';
-export * from './session_pool/index.js';
 export * from './storages/index.js';
-export * from './validators.js';
-export * from './cookie_utils.js';
-export * from './recoverable_state.js';
-export { PseudoUrl } from '@apify/pseudo_url';
-//# sourceMappingURL=index.js.map
+export * from './memory-storage/index.js';
+// Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no
+// semver guarantees and must not reach the public surface. Internal consumers import them directly.
+export { ArgumentValidationError, validators } from './validators.js';
+export * from './recoverable-state.js';
+export { EnqueueStrategy } from '@crawlee/utils';

package/internal.d.ts

@@ -0,0 +1,8 @@
+export * from './iterables.js';
+export * from './storages/batched-adds.js';
+export { joinRequestSourceStatuses } from './storages/request-loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/internal.js

@@ -0,0 +1,9 @@
+// Not part of the public API: internals that other Crawlee packages need but that carry no semver guarantees.
+export * from './iterables.js';
+export * from './storages/batched-adds.js';
+export { joinRequestSourceStatuses } from './storages/request-loader.js';
+export * from './system-info/cpu-info.js';
+export * from './system-info/memory-info.js';
+export * from './system-info/ps-tree.js';
+export * from './system-info/runtime.js';
+export * from './url.js';

package/iterables.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+    /**
+     * Peeks at the next value without consuming it from the iterator.
+     * Subsequent calls to peek() will return the same value until next() is called.
+     *
+     * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+     */
+    peek(): Promise<T | undefined>;
+}
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+    [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;