@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/events/event_manager.js

@@ -1,7 +1,6 @@
 import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
-import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
+import { serviceLocator } from '../service_locator.js';
 export var EventType;
 (function (EventType) {
     EventType["PERSIST_STATE"] = "persistState";
@@ -9,30 +8,31 @@ export var EventType;
     EventType["MIGRATING"] = "migrating";
     EventType["ABORTING"] = "aborting";
     EventType["EXIT"] = "exit";
+    EventType["STATUS_MESSAGE"] = "statusMessage";
 })(EventType || (EventType = {}));
 export class EventManager {
-    config;
     events = new AsyncEventEmitter();
     initialized = false;
     intervals = {};
-    log = log.child({ prefix: 'Events' });
-    constructor(config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.events.setMaxListeners(50);
+    log = serviceLocator.getLogger().child({ prefix: 'Events' });
+    #persistStateIntervalMillis;
+    constructor(options) {
+        this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
+        // One MIGRATING listener per RequestQueue, and ThrottlingRequestManager opens one per domain.
+        this.events.setMaxListeners(150);
     }
     /**
-     * Initializes the event manager by creating the `persistState` event interval.
+     * Initializes the event manager by starting the `persistState` event interval.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     async init() {
         if (this.initialized) {
             return;
         }
-        const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
         this.intervals.persistState = betterSetInterval((intervalCallback) => {
-            this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+            this.emit(EventType.PERSIST_STATE, { isMigrating: false });
             intervalCallback();
-        }, persistStateIntervalMillis);
+        }, this.#persistStateIntervalMillis);
         this.initialized = true;
     }
     /**
@@ -46,7 +46,7 @@ export class EventManager {
         betterClearInterval(this.intervals.persistState);
         this.initialized = false;
         // Emit final PERSIST_STATE event
-        this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
+        this.emit(EventType.PERSIST_STATE, { isMigrating: false });
         // Wait for PERSIST_STATE to process
         await this.waitForAllListenersToComplete();
     }
@@ -86,4 +86,3 @@ export class EventManager {
         return this.events.waitForAllListenersToComplete();
     }
 }
-//# sourceMappingURL=event_manager.js.map

package/events/index.d.ts

@@ -1,3 +1,2 @@
 export * from './event_manager.js';
 export * from './local_event_manager.js';
-//# sourceMappingURL=index.d.ts.map

package/events/index.js

@@ -1,3 +1,2 @@
 export * from './event_manager.js';
 export * from './local_event_manager.js';
-//# sourceMappingURL=index.js.map

package/events/local_event_manager.d.ts

@@ -1,7 +1,19 @@
-import { EventManager } from './event_manager.js';
+import type { Configuration } from '../configuration.js';
+import { EventManager, type EventManagerOptions } from './event_manager.js';
+export interface LocalEventManagerOptions extends EventManagerOptions {
+    /** Interval between emitted `systemInfo` events in milliseconds. */
+    systemInfoIntervalMillis: number;
+}
 export declare class LocalEventManager extends EventManager {
+    #private;
+    constructor(options: LocalEventManagerOptions);
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfiguration(configuration?: Configuration): LocalEventManager;
     /**
-     * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     init(): Promise<void>;
@@ -23,5 +35,5 @@ export declare class LocalEventManager extends EventManager {
     private createSystemInfo;
     private createCpuInfo;
     private createMemoryInfo;
+    private getMemoryInfo;
 }
-//# sourceMappingURL=local_event_manager.d.ts.map

package/events/local_event_manager.js

@@ -1,10 +1,25 @@
-import { getCurrentCpuTicksV2, getMemoryInfo, isContainerized } from '@crawlee/utils';
-import log from '@apify/log';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { EventManager } from './event_manager.js';
+import { serviceLocator } from '../service_locator.js';
+import { EventManager, EventType } from './event_manager.js';
 export class LocalEventManager extends EventManager {
+    #systemInfoIntervalMillis;
+    constructor(options) {
+        super(options);
+        this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+    }
+    /**
+     * Creates a new `LocalEventManager` based on the provided `Configuration`.
+     * Uses the global configuration from the service locator if none is provided.
+     */
+    static fromConfiguration(configuration) {
+        const resolvedConfiguration = configuration ?? serviceLocator.getConfiguration();
+        return new LocalEventManager({
+            persistStateIntervalMillis: resolvedConfiguration.persistStateIntervalMillis,
+            systemInfoIntervalMillis: resolvedConfiguration.systemInfoIntervalMillis,
+        });
+    }
     /**
-     * Initializes the EventManager and sets up periodic `systemInfo` and `persistState` events.
+     * Initializes the EventManager and sets up periodic `systemInfo` events.
      * This is automatically called at the beginning of `crawler.run()`.
      */
     async init() {
@@ -12,9 +27,8 @@ export class LocalEventManager extends EventManager {
             return;
         }
         await super.init();
-        const systemInfoIntervalMillis = this.config.get('systemInfoIntervalMillis');
         this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
-        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), systemInfoIntervalMillis);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.#systemInfoIntervalMillis);
     }
     /**
      * @inheritDoc
@@ -31,16 +45,17 @@ export class LocalEventManager extends EventManager {
      */
     async emitSystemInfoEvent(intervalCallback) {
         const info = await this.createSystemInfo({
-            maxUsedCpuRatio: this.config.get('maxUsedCpuRatio'),
+            maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
         });
-        this.events.emit("systemInfo" /* EventType.SYSTEM_INFO */, info);
+        this.events.emit(EventType.SYSTEM_INFO, info);
         intervalCallback();
     }
     /**
      * @internal
      */
     async isContainerizedWrapper() {
-        return this.config.get('containerized', await isContainerized());
+        const { isContainerized } = await import('../system-info/runtime.js');
+        return serviceLocator.getConfiguration().containerized ?? (await isContainerized());
     }
     /**
      * Creates a SystemInfo object based on local metrics.
@@ -53,7 +68,11 @@ export class LocalEventManager extends EventManager {
         };
     }
     async createCpuInfo(options) {
-        const usedCpuRatio = await getCurrentCpuTicksV2(await this.isContainerizedWrapper());
+        const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
+        const usedCpuRatio = await getCurrentCpuTicksV2({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
         return {
             cpuCurrentUsage: usedCpuRatio * 100,
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
@@ -61,15 +80,22 @@ export class LocalEventManager extends EventManager {
     }
     async createMemoryInfo() {
         try {
-            const memInfo = await getMemoryInfo(await this.isContainerizedWrapper());
+            const memInfo = await this.getMemoryInfo();
             return {
+                memTotalBytes: memInfo.totalBytes,
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
             };
         }
         catch (err) {
-            log.exception(err, 'Memory snapshot failed.');
+            this.log.exception(err, 'Memory snapshot failed.');
             return {};
         }
     }
+    async getMemoryInfo() {
+        const { getMemoryInfo } = await import('../system-info/memory-info.js');
+        return getMemoryInfo({
+            containerized: await this.isContainerizedWrapper(),
+            logger: serviceLocator.getLogger(),
+        });
+    }
 }
-//# sourceMappingURL=local_event_manager.js.map

package/http.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export declare function parseRetryAfterHeader(value?: string | null): number | null;

package/http.js

@@ -0,0 +1,28 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export function parseRetryAfterHeader(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim();
+    // Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
+    if (/^\d+$/.test(trimmed)) {
+        // `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
+        // as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
+        // caller would defer the request for free and re-send it immediately.
+        const delayMs = Number(trimmed) * 1000;
+        return delayMs > 0 ? delayMs : null;
+    }
+    const date = Date.parse(trimmed);
+    if (!Number.isNaN(date)) {
+        const delayMs = date - Date.now();
+        return delayMs > 0 ? delayMs : null;
+    }
+    return null;
+}

package/index.d.ts

@@ -1,19 +1,22 @@
+export * from './debug.js';
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
+export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
 export * from './router.js';
 export * from './serialization.js';
 export * from './session_pool/index.js';
 export * from './storages/index.js';
+export * from './memory-storage/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
-export { PseudoUrl } from '@apify/pseudo_url';
-export { Dictionary, Awaitable, Constructor, StorageClient, Cookie, QueueOperationInfo } from '@crawlee/types';
-//# sourceMappingURL=index.d.ts.map
+export * from './http.js';
+export * from './recoverable_state.js';
+export type { StorageBackend } from '@crawlee/types';

package/index.js

@@ -1,18 +1,21 @@
+export * from './debug.js';
 export * from './errors.js';
 export * from './autoscaling/index.js';
 export * from './configuration.js';
+export * from './service_locator.js';
 export * from './crawlers/index.js';
 export * from './enqueue_links/index.js';
 export * from './events/index.js';
-export * from './http_clients/index.js';
 export * from './log.js';
+export * from './owned_or_injected.js';
 export * from './proxy_configuration.js';
 export * from './request.js';
 export * from './router.js';
 export * from './serialization.js';
 export * from './session_pool/index.js';
 export * from './storages/index.js';
+export * from './memory-storage/index.js';
 export * from './validators.js';
 export * from './cookie_utils.js';
-export { PseudoUrl } from '@apify/pseudo_url';
-//# sourceMappingURL=index.js.map
+export * from './http.js';
+export * from './recoverable_state.js';

package/iterables.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
+/**
+ * An async iterator that also supports peeking at the next value without consuming it.
+ * Extends both AsyncIterator and AsyncIterable interfaces.
+ * @internal
+ */
+export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
+    /**
+     * Peeks at the next value without consuming it from the iterator.
+     * Subsequent calls to peek() will return the same value until next() is called.
+     *
+     * @returns Promise that resolves to the next value, or undefined if the iterator is exhausted
+     */
+    peek(): Promise<T | undefined>;
+}
+/**
+ * An async iterable that yields peekable async iterators.
+ * @internal
+ */
+export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
+    [Symbol.asyncIterator](): PeekableAsyncIterator<T>;
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export declare function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;

package/iterables.js

@@ -0,0 +1,134 @@
+import { inspect } from 'node:util';
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable(iterable) {
+    yield* iterable;
+}
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable(iterable, chunkSize) {
+    const getChunkSize = typeof chunkSize === 'function' ? chunkSize : () => chunkSize;
+    if (typeof chunkSize === 'number' && chunkSize < 1) {
+        throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+    }
+    const iterator = Symbol.asyncIterator in iterable
+        ? iterable[Symbol.asyncIterator]()
+        : iterable[Symbol.iterator]();
+    while (true) {
+        const currentSize = getChunkSize();
+        if (currentSize < 1)
+            break;
+        const chunk = [];
+        for (let i = 0; i < currentSize; i++) {
+            const next = await iterator.next();
+            if (next.done) {
+                break;
+            }
+            chunk.push(next.value);
+        }
+        if (chunk.length === 0)
+            break;
+        yield chunk;
+    }
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable(iterable) {
+    const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+    let peekedValue;
+    let isExhausted = false;
+    const peekableIterator = {
+        async next() {
+            // If we have peeked a value, return it and clear the peek
+            if (peekedValue !== undefined) {
+                const result = peekedValue;
+                peekedValue = undefined;
+                if (result.done) {
+                    isExhausted = true;
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value: result.value };
+            }
+            if (isExhausted) {
+                return { done: true, value: undefined };
+            }
+            const result = await iterator.next();
+            if (result.done) {
+                isExhausted = true;
+            }
+            return result;
+        },
+        async peek() {
+            if (peekedValue !== undefined) {
+                return peekedValue.done ? undefined : peekedValue.value;
+            }
+            if (isExhausted) {
+                return undefined;
+            }
+            const result = await iterator.next();
+            peekedValue = { done: result.done ?? false, value: result.value };
+            if (result.done) {
+                isExhausted = true;
+                return undefined;
+            }
+            return result.value;
+        },
+        [Symbol.asyncIterator]() {
+            return this;
+        },
+    };
+    return {
+        [Symbol.asyncIterator]() {
+            return peekableIterator;
+        },
+    };
+}

package/log.d.ts

@@ -1,3 +1,82 @@
-import log, { Log, Logger, LoggerJson, LoggerOptions, LoggerText, LogLevel } from '@apify/log';
-export { log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText };
-//# sourceMappingURL=log.d.ts.map
+import type { CrawleeLogger, CrawleeLoggerOptions } from '@crawlee/types';
+import type { LoggerOptions } from '@apify/log';
+import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+export type { CrawleeLogger, CrawleeLoggerOptions };
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
+    private options;
+    private readonly warningsLogged;
+    constructor(options?: Partial<CrawleeLoggerOptions>);
+    /**
+     * Core logging method. Subclasses must implement this to dispatch log messages
+     * to the underlying logger (Winston, Pino, console, etc.).
+     *
+     * Level filtering is the responsibility of the underlying library — this method
+     * is called for every message regardless of the current level.
+     *
+     * @param level Crawlee log level (use {@link LogLevel} constants)
+     * @param message The log message
+     * @param data Optional structured data to attach to the log entry
+     */
+    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    /**
+     * Creates a child logger instance. Subclasses must implement this to define
+     * how child loggers are created for the underlying logger.
+     */
+    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    getOptions(): CrawleeLoggerOptions;
+    setOptions(options: Partial<CrawleeLoggerOptions>): void;
+    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+    error(message: string, data?: Record<string, unknown>): void;
+    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
+    softFail(message: string, data?: Record<string, unknown>): void;
+    warning(message: string, data?: Record<string, unknown>): void;
+    warningOnce(message: string): void;
+    info(message: string, data?: Record<string, unknown>): void;
+    debug(message: string, data?: Record<string, unknown>): void;
+    perf(message: string, data?: Record<string, unknown>): void;
+    deprecated(message: string): void;
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export declare class ApifyLogAdapter extends BaseCrawleeLogger {
+    private readonly apifyLog;
+    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
+    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
+    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
+}
+export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
+export type { LoggerOptions };