@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.81

package/autoscaling/load_signal.d.ts

@@ -0,0 +1,99 @@
+import type { EventManager, EventTypeName } from '../events/event_manager.js';
+import type { ClientInfo } from './system_status.js';
+/**
+ * A snapshot of a resource's overload state at a point in time.
+ */
+export interface LoadSnapshot {
+    createdAt: Date;
+    isOverloaded: boolean;
+}
+/**
+ * A signal that reports whether a particular resource is overloaded.
+ *
+ * `SystemStatus` aggregates multiple `LoadSignal` instances to determine
+ * overall system health. The built-in signals cover memory, CPU, event loop,
+ * and API client rate limits. You can implement this interface to add
+ * custom overload signals (e.g. navigation timeouts, proxy health).
+ */
+export interface LoadSignal {
+    /** Human-readable name used in logging and `SystemInfo` keys. */
+    readonly name: string;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before the signal
+     * is considered overloaded. For example, `0.2` means the signal fires
+     * when more than 20% of the sample window is overloaded.
+     */
+    readonly overloadedRatio: number;
+    /** Start collecting snapshots. Called when the pool starts. */
+    start(): Promise<void>;
+    /** Stop collecting snapshots. Called when the pool shuts down. */
+    stop(): Promise<void>;
+    /**
+     * Return snapshots for a recent time window (used for "current" status).
+     * @param sampleDurationMillis How far back to look, in milliseconds.
+     */
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+}
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
+ * Signals compose with this instead of inheriting from a base class.
+ */
+export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
+    private snapshots;
+    private readonly historyMillis;
+    constructor(historyMillis?: number);
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot: T, now?: Date): void;
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis?: number): T[];
+    /**
+     * Direct access to the underlying array (for backward-compat getters).
+     */
+    getAll(): T[];
+    /**
+     * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
+     *
+     * The `handler` receives the store (to read previous snapshots) and the
+     * interval callback (which it **must** call when done). It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromInterval<T extends LoadSnapshot>(options: {
+        name: string;
+        overloadedRatio: number;
+        intervalMillis: number;
+        snapshotHistoryMillis?: number;
+        handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
+    }): Omit<LoadSignal, 'getSample'> & {
+        store: SnapshotStore<T>;
+        handle: (cb: () => unknown) => void;
+        getSample(sampleDurationMillis?: number): T[];
+    };
+    /**
+     * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
+     *
+     * The `handler` receives the event payload and the store. It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromEvent<T extends LoadSnapshot, E>(options: {
+        name: string;
+        overloadedRatio: number;
+        events: EventManager;
+        event: EventTypeName;
+        snapshotHistoryMillis?: number;
+        handler: (store: SnapshotStore<T>, payload: E) => void;
+    }): Omit<LoadSignal, 'getSample'> & {
+        store: SnapshotStore<T>;
+        handle: (payload: E) => void;
+        getSample(sampleDurationMillis?: number): T[];
+    };
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ */
+export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;

package/autoscaling/load_signal.js

@@ -0,0 +1,135 @@
+import { weightedAvg } from '@crawlee/utils';
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
+ * Signals compose with this instead of inheriting from a base class.
+ */
+export class SnapshotStore {
+    snapshots = [];
+    historyMillis;
+    constructor(historyMillis = 30_000) {
+        this.historyMillis = historyMillis;
+    }
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot, now = snapshot.createdAt) {
+        // Inline pruning to avoid private-method transpilation issues
+        let oldCount = 0;
+        for (let i = 0; i < this.snapshots.length; i++) {
+            const { createdAt } = this.snapshots[i];
+            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
+                oldCount++;
+            else
+                break;
+        }
+        if (oldCount)
+            this.snapshots.splice(0, oldCount);
+        this.snapshots.push(snapshot);
+    }
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis) {
+        if (!sampleDurationMillis)
+            return this.snapshots;
+        const sample = [];
+        let idx = this.snapshots.length;
+        if (!idx)
+            return sample;
+        const latestTime = this.snapshots[idx - 1].createdAt;
+        while (idx--) {
+            const snapshot = this.snapshots[idx];
+            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
+                sample.unshift(snapshot);
+            }
+            else {
+                break;
+            }
+        }
+        return sample;
+    }
+    /**
+     * Direct access to the underlying array (for backward-compat getters).
+     */
+    getAll() {
+        return this.snapshots;
+    }
+    /**
+     * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
+     *
+     * The `handler` receives the store (to read previous snapshots) and the
+     * interval callback (which it **must** call when done). It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromInterval(options) {
+        const store = new SnapshotStore(options.snapshotHistoryMillis);
+        let interval = null;
+        const handle = (cb) => options.handler(store, cb);
+        return {
+            name: options.name,
+            overloadedRatio: options.overloadedRatio,
+            store,
+            handle,
+            getSample: (ms) => store.getSample(ms),
+            async start() {
+                interval = betterSetInterval(handle, options.intervalMillis);
+            },
+            async stop() {
+                betterClearInterval(interval);
+            },
+        };
+    }
+    /**
+     * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
+     *
+     * The `handler` receives the event payload and the store. It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromEvent(options) {
+        const store = new SnapshotStore(options.snapshotHistoryMillis);
+        const handle = (payload) => options.handler(store, payload);
+        return {
+            name: options.name,
+            overloadedRatio: options.overloadedRatio,
+            store,
+            handle,
+            getSample: (ms) => store.getSample(ms),
+            async start() {
+                options.events.on(options.event, handle);
+            },
+            async stop() {
+                options.events.off(options.event, handle);
+            },
+        };
+    }
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ */
+export function evaluateLoadSignalSample(sample, overloadedRatio) {
+    if (sample.length === 0) {
+        return {
+            isOverloaded: false,
+            limitRatio: overloadedRatio,
+            actualRatio: 0,
+        };
+    }
+    const weights = [];
+    const values = [];
+    for (let i = 1; i < sample.length; i++) {
+        const previous = sample[i - 1];
+        const current = sample[i];
+        const weight = +current.createdAt - +previous.createdAt;
+        weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
+        values.push(+current.isOverloaded);
+    }
+    const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
+    return {
+        isOverloaded: wAvg > overloadedRatio,
+        limitRatio: overloadedRatio,
+        actualRatio: Math.round(wAvg * 1000) / 1000,
+    };
+}

package/autoscaling/memory_load_signal.d.ts

@@ -0,0 +1,43 @@
+import type { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
+import type { LoadSignal, LoadSnapshot } from './load_signal.js';
+import type { SystemInfo } from './system_status.js';
+export interface MemorySnapshot extends LoadSnapshot {
+    usedBytes?: number;
+}
+export interface MemoryLoadSignalOptions {
+    maxUsedMemoryRatio?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+    config: Configuration;
+    log?: CrawleeLogger;
+}
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
+ * the used-to-available memory ratio exceeds a threshold.
+ */
+export declare class MemoryLoadSignal implements LoadSignal {
+    readonly name = "memInfo";
+    readonly overloadedRatio: number;
+    private readonly store;
+    private readonly config;
+    private readonly events;
+    private readonly log;
+    private readonly maxUsedMemoryRatio;
+    private maxMemoryRatio;
+    private maxMemoryBytes;
+    private lastLoggedCriticalMemoryOverloadAt;
+    constructor(options: MemoryLoadSignalOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): MemorySnapshot[];
+    /**
+     * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
+     */
+    getMemorySnapshots(): MemorySnapshot[];
+    /** @internal */
+    _onSystemInfo(systemInfo: SystemInfo): void;
+    /** @internal */
+    _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
+    private _getTotalMemoryBytes;
+}

package/autoscaling/memory_load_signal.js

@@ -0,0 +1,102 @@
+import { serviceLocator } from '../service_locator.js';
+import { getMemoryInfo } from '../system-info/memory-info.js';
+import { isContainerized } from '../system-info/runtime.js';
+import { SnapshotStore } from './load_signal.js';
+const RESERVE_MEMORY_RATIO = 0.5;
+const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
+ * the used-to-available memory ratio exceeds a threshold.
+ */
+export class MemoryLoadSignal {
+    name = 'memInfo';
+    overloadedRatio;
+    store;
+    config;
+    events;
+    log;
+    maxUsedMemoryRatio;
+    maxMemoryRatio;
+    maxMemoryBytes;
+    lastLoggedCriticalMemoryOverloadAt = null;
+    constructor(options) {
+        this.store = new SnapshotStore(options.snapshotHistoryMillis);
+        this.config = options.config;
+        this.events = serviceLocator.getEventManager();
+        this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
+        this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
+        this.overloadedRatio = options.overloadedRatio ?? 0.2;
+        this._onSystemInfo = this._onSystemInfo.bind(this);
+    }
+    async start() {
+        const memoryMbytes = this.config.memoryMbytes ?? 0;
+        if (memoryMbytes > 0) {
+            this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
+        }
+        else {
+            this.maxMemoryRatio = this.config.availableMemoryRatio;
+            if (!this.maxMemoryRatio) {
+                throw new Error('availableMemoryRatio is not set in configuration.');
+            }
+            else {
+                this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
+                    'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
+            }
+            // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
+            this.maxMemoryBytes = await this._getTotalMemoryBytes();
+        }
+        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
+    }
+    async stop() {
+        this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
+    }
+    getSample(sampleDurationMillis) {
+        return this.store.getSample(sampleDurationMillis);
+    }
+    /**
+     * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
+     */
+    getMemorySnapshots() {
+        return this.store.getAll();
+    }
+    /** @internal */
+    _onSystemInfo(systemInfo) {
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        const { memCurrentBytes, memTotalBytes } = systemInfo;
+        let maxMemoryBytes = this.maxMemoryBytes;
+        if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
+            maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
+        }
+        const snapshot = {
+            createdAt,
+            isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
+            usedBytes: memCurrentBytes,
+        };
+        this.store.push(snapshot, createdAt);
+        this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
+    }
+    /** @internal */
+    _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
+        const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
+        const { memCurrentBytes } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        if (this.lastLoggedCriticalMemoryOverloadAt &&
+            +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+            return;
+        const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
+        const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
+        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
+        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
+        if (isCriticalOverload) {
+            const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
+            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
+            this.log.warning('Memory is critically overloaded. ' +
+                `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
+            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
+        }
+    }
+    async _getTotalMemoryBytes() {
+        const containerized = this.config.containerized ?? (await isContainerized());
+        return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
+    }
+}

package/autoscaling/snapshotter.d.ts

@@ -1,9 +1,11 @@
-import type { StorageClient } from '@crawlee/types';
-import type { Log } from '@apify/log';
-import type { BetterIntervalID } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
-import type { EventManager } from '../events/event_manager.js';
-import type { SystemInfo } from './system_status.js';
+import type { StorageBackend } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
+import type { ClientSnapshot } from './client_load_signal.js';
+import type { CpuSnapshot } from './cpu_load_signal.js';
+import type { EventLoopSnapshot } from './event_loop_load_signal.js';
+import type { LoadSignal } from './load_signal.js';
+import type { MemorySnapshot } from './memory_load_signal.js';
 export interface SnapshotterOptions {
     /**
      * Defines the interval of measuring the event loop response time.
@@ -31,46 +33,22 @@ export interface SnapshotterOptions {
     /**
      * Defines the maximum number of new rate limit errors within
      * the given interval.
-     * @default 1
+     * @default 3
      */
     maxClientErrors?: number;
     /**
      * Sets the interval in seconds for which a history of resource snapshots
      * will be kept. Increasing this to very high numbers will affect performance.
-     * @default 60
+     * @default 30
      */
     snapshotHistorySecs?: number;
     /** @internal */
-    log?: Log;
+    log?: CrawleeLogger;
     /** @internal */
-    client?: StorageClient;
+    client?: StorageBackend;
     /** @internal */
     config?: Configuration;
 }
-interface MemorySnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedBytes?: number;
-}
-interface CpuSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedRatio: number;
-    ticks?: {
-        idle: number;
-        total: number;
-    };
-}
-interface EventLoopSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    exceededMillis: number;
-}
-interface ClientSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    rateLimitErrorCount: number;
-}
 /**
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -94,27 +72,26 @@ interface ClientSnapshot {
  *
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
  * typically received from the request queue, exceed the set limit within the set interval.
+ *
  * @category Scaling
  */
 export declare class Snapshotter {
-    log: Log;
-    client: StorageClient;
-    config: Configuration;
-    events: EventManager;
-    eventLoopSnapshotIntervalMillis: number;
-    clientSnapshotIntervalMillis: number;
-    snapshotHistoryMillis: number;
-    maxBlockedMillis: number;
-    maxUsedMemoryRatio: number;
-    maxClientErrors: number;
-    maxMemoryBytes: number;
-    cpuSnapshots: CpuSnapshot[];
-    eventLoopSnapshots: EventLoopSnapshot[];
-    memorySnapshots: MemorySnapshot[];
-    clientSnapshots: ClientSnapshot[];
-    eventLoopInterval: BetterIntervalID;
-    clientInterval: BetterIntervalID;
-    lastLoggedCriticalMemoryOverloadAt: Date | null;
+    readonly log: CrawleeLogger;
+    readonly client: StorageBackend;
+    readonly config: Configuration;
+    private readonly memorySignal;
+    private readonly eventLoopSignal;
+    private readonly cpuSignal;
+    private readonly clientSignal;
+    /**
+     * Returns the four built-in signals as an array, so `SystemStatus` can
+     * iterate them alongside any custom `LoadSignal` instances.
+     */
+    getLoadSignals(): LoadSignal[];
+    get cpuSnapshots(): CpuSnapshot[];
+    get eventLoopSnapshots(): EventLoopSnapshot[];
+    get memorySnapshots(): MemorySnapshot[];
+    get clientSnapshots(): ClientSnapshot[];
     /**
      * @param [options] All `Snapshotter` configuration options.
      */
@@ -147,43 +124,4 @@ export declare class Snapshotter {
      * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
      */
     getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
-    /**
-     * Finds the latest snapshots by sampleDurationMillis in the provided array.
-     */
-    protected _getSample<T extends {
-        createdAt: Date;
-    }>(snapshots: T[], sampleDurationMillis?: number): T[];
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
-     */
-    protected _snapshotMemory(systemInfo: SystemInfo): void;
-    /**
-     * Checks for critical memory overload and logs it to the console.
-     */
-    protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
-    /**
-     * Creates a snapshot of current event loop delay.
-     */
-    protected _snapshotEventLoop(intervalCallback: () => unknown): void;
-    /**
-     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
-     */
-    protected _snapshotCpu(systemInfo: SystemInfo): void;
-    /**
-     * Creates a snapshot of current API state by checking for
-     * rate limit errors. Only errors produced by a 2nd retry
-     * of the API call are considered for snapshotting since
-     * earlier errors may just be caused by a random spike in
-     * number of requests and do not necessarily signify API
-     * overloading.
-     */
-    protected _snapshotClient(intervalCallback: () => unknown): void;
-    /**
-     * Removes snapshots that are older than the snapshotHistorySecs option
-     * from the array (destructively - in place).
-     */
-    protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
 }
-export {};
-//# sourceMappingURL=snapshotter.d.ts.map