@crawlee/core 4.0.0-beta.9 → 4.0.0-beta.91

package/autoscaling/cpu_load_signal.js

@@ -0,0 +1,46 @@
+import { serviceLocator } from '../service_locator.js';
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when the platform or local OS metrics indicate the
+ * CPU is overloaded.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export class CpuLoadSignal {
+    name = 'cpuInfo';
+    overloadedRatio;
+    store = new SnapshotStore();
+    events;
+    constructor(options = {}) {
+        this.overloadedRatio = options.overloadedRatio ?? 0.4;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
+        this.store.clear();
+        // Resolved here rather than in the constructor, so an instance built ahead of time (to be wrapped, or shared
+        // between systems) cannot capture whichever event manager happened to be registered at that moment.
+        this.events = serviceLocator.getEventManager();
+        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+    }
+    async stop() {
+        this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+        this.events = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.store.getSample(sampleDurationMillis);
+    }
+    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
+    handle(systemInfo) {
+        const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        this.store.push({
+            createdAt,
+            isOverloaded: isCpuOverloaded,
+            usedRatio: Math.ceil(cpuCurrentUsage / 100),
+        }, createdAt);
+    }
+}

package/autoscaling/event_loop_load_signal.d.ts

@@ -0,0 +1,54 @@
+import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
+/**
+ * A snapshot produced by the built-in event loop signal.
+ * @internal
+ */
+export interface EventLoopSnapshot extends LoadSnapshot {
+    exceededMillis: number;
+}
+/**
+ * Tuning for the built-in **event loop** load signal, as accepted both by {@link EventLoopLoadSignal} and by the
+ * {@link LoadSignalsOptions.eventLoop|`eventLoop`} shorthand on {@link LoadSignalsOptions}.
+ */
+export interface EventLoopLoadSignalOptions {
+    /**
+     * Defines the interval of measuring the event loop response time, in seconds.
+     * @default 0.5
+     */
+    snapshotIntervalSecs?: number;
+    /**
+     * Maximum allowed delay of the event loop in milliseconds.
+     * Exceeding this limit overloads the event loop.
+     * @default 50
+     */
+    maxBlockedMillis?: number;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before the event loop counts as overloaded.
+     * @default 0.6
+     */
+    overloadedRatio?: number;
+}
+/**
+ * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export declare class EventLoopLoadSignal implements LoadSignal {
+    readonly name = "eventLoopInfo";
+    readonly overloadedRatio: number;
+    private readonly store;
+    private readonly intervalMillis;
+    private readonly maxBlockedMillis;
+    private interval?;
+    constructor(options?: EventLoopLoadSignalOptions);
+    start(context: LoadSignalStartContext): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+    /**
+     * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
+     * @internal Also lets tests drive the measurement without waiting on a timer.
+     */
+    handle(intervalCallback: () => unknown): void;
+}

package/autoscaling/event_loop_load_signal.js

@@ -0,0 +1,60 @@
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Periodically measures event loop delay and reports overload when the delay exceeds a configured threshold.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export class EventLoopLoadSignal {
+    name = 'eventLoopInfo';
+    overloadedRatio;
+    store = new SnapshotStore();
+    intervalMillis;
+    maxBlockedMillis;
+    interval;
+    constructor(options = {}) {
+        this.overloadedRatio = options.overloadedRatio ?? 0.6;
+        this.intervalMillis = (options.snapshotIntervalSecs ?? 0.5) * 1000;
+        this.maxBlockedMillis = options.maxBlockedMillis ?? 50;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, or the downtime gets charged to the event loop: `handle()` measures
+        // the gap since the previous snapshot, which across a restart is however long the system was stopped.
+        this.store.clear();
+        this.interval = betterSetInterval(this.handle, this.intervalMillis);
+    }
+    async stop() {
+        if (this.interval)
+            betterClearInterval(this.interval);
+        this.interval = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.store.getSample(sampleDurationMillis);
+    }
+    /**
+     * Records one snapshot: how much later than scheduled this tick ran is how long the loop was blocked.
+     * @internal Also lets tests drive the measurement without waiting on a timer.
+     */
+    handle(intervalCallback) {
+        const now = new Date();
+        const snapshot = {
+            createdAt: now,
+            isOverloaded: false,
+            exceededMillis: 0,
+        };
+        const all = this.store.getAll();
+        const previousSnapshot = all[all.length - 1];
+        if (previousSnapshot) {
+            const delta = now.getTime() - +previousSnapshot.createdAt - this.intervalMillis;
+            if (delta > this.maxBlockedMillis)
+                snapshot.isOverloaded = true;
+            snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
+        }
+        this.store.push(snapshot, now);
+        intervalCallback();
+    }
+}

package/autoscaling/index.d.ts

@@ -1,4 +1,9 @@
 export * from './autoscaled_pool.js';
+export * from './concurrency_system.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.d.ts.map

package/autoscaling/index.js

@@ -1,4 +1,9 @@
 export * from './autoscaled_pool.js';
+export * from './concurrency_system.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.js.map

package/autoscaling/load_signal.d.ts

@@ -0,0 +1,99 @@
+import type { ClientInfo } from './system_status.js';
+/**
+ * A snapshot of a resource's overload state at a point in time.
+ */
+export interface LoadSnapshot {
+    createdAt: Date;
+    isOverloaded: boolean;
+}
+/**
+ * Handed to a {@link LoadSignal} when it starts, so it can size its snapshot retention to what it will actually
+ * be asked for — without having to know how the {@link ConcurrencySystem} that drives it is configured.
+ */
+export interface LoadSignalStartContext {
+    /**
+     * The longest sample window the signal will be queried with (the wider of the task-gating and autoscaling
+     * windows). Keeping less history than this contributes a narrower view of the resource than the other signals;
+     * keeping more is wasted memory, as the extra snapshots are never sampled.
+     */
+    maxSampleWindowMillis: number;
+}
+/**
+ * A signal that reports whether a particular resource is overloaded. The {@link ConcurrencySystem} aggregates
+ * several of them — if any one reports overload, the system is overloaded.
+ *
+ * The built-in signals cover memory, CPU, event loop and storage-client rate limits. Implement this interface to add
+ * your own (navigation timeouts, proxy health, …) and pass them via
+ * {@link LoadSignalsOptions.custom|`loadSignals.custom`}; {@link SnapshotStore} does the time-windowed
+ * bookkeeping if you want it. Each built-in is also a public class, so one can be *wrapped* rather than reimplemented
+ * — construct it yourself and switch the default off with {@link LoadSignalsOptions.cpu|`cpu: false`} or friends.
+ */
+export interface LoadSignal {
+    /**
+     * This signal's key in the reported {@link SystemInfo}, also used in logging — so it must be unique among the
+     * signals of one {@link ConcurrencySystem}, which throws on a duplicate. The four built-in names (`memInfo`,
+     * `eventLoopInfo`, `cpuInfo`, `clientInfo`) land in the correspondingly named `SystemInfo` fields rather than the
+     * `loadSignalInfo` bag; taking one over means switching that built-in off.
+     */
+    readonly name: string;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before the signal
+     * is considered overloaded. For example, `0.2` means the signal fires
+     * when more than 20% of the sample window is overloaded.
+     */
+    readonly overloadedRatio: number;
+    /**
+     * Start collecting snapshots, retaining at least the sample window named in the `context`. Called when the
+     * {@link ConcurrencySystem} starts — which may be a *restart*, so drop anything measured before it.
+     */
+    start(context: LoadSignalStartContext): Promise<void>;
+    /** Stop collecting snapshots. Called when the {@link ConcurrencySystem} shuts down. */
+    stop(): Promise<void>;
+    /**
+     * Return snapshots for a recent time window (used for "current" status).
+     * @param sampleDurationMillis How far back to look, in milliseconds.
+     */
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+}
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
+ * and so can yours — it is the only part of their machinery worth reusing.
+ */
+export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
+    private snapshots;
+    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
+    private historyMillis;
+    /**
+     * Sizes retention to the window the signal will be sampled over, as handed to it in
+     * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
+     * its start context grows unboundedly.
+     */
+    useSampleWindow(maxSampleWindowMillis: number): void;
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot: T, now?: Date): void;
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis?: number): T[];
+    /**
+     * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
+     * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
+     */
+    getAll(): T[];
+    /**
+     * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
+     * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
+     * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
+     * rather than on stop leaves a finished session readable.
+     */
+    clear(): void;
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ * @internal
+ */
+export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;

package/autoscaling/load_signal.js

@@ -0,0 +1,103 @@
+import { weightedAvg } from '@crawlee/utils';
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
+ * and so can yours — it is the only part of their machinery worth reusing.
+ */
+export class SnapshotStore {
+    snapshots = [];
+    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
+    historyMillis = Infinity;
+    /**
+     * Sizes retention to the window the signal will be sampled over, as handed to it in
+     * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
+     * its start context grows unboundedly.
+     */
+    useSampleWindow(maxSampleWindowMillis) {
+        this.historyMillis = maxSampleWindowMillis;
+    }
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot, now = snapshot.createdAt) {
+        // Inline pruning to avoid private-method transpilation issues
+        let oldCount = 0;
+        for (let i = 0; i < this.snapshots.length; i++) {
+            const { createdAt } = this.snapshots[i];
+            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
+                oldCount++;
+            else
+                break;
+        }
+        if (oldCount)
+            this.snapshots.splice(0, oldCount);
+        this.snapshots.push(snapshot);
+    }
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis) {
+        if (!sampleDurationMillis)
+            return this.snapshots;
+        const sample = [];
+        let idx = this.snapshots.length;
+        if (!idx)
+            return sample;
+        const latestTime = this.snapshots[idx - 1].createdAt;
+        while (idx--) {
+            const snapshot = this.snapshots[idx];
+            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
+                sample.unshift(snapshot);
+            }
+            else {
+                break;
+            }
+        }
+        return sample;
+    }
+    /**
+     * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
+     * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
+     */
+    getAll() {
+        return this.snapshots;
+    }
+    /**
+     * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
+     * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
+     * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
+     * rather than on stop leaves a finished session readable.
+     */
+    clear() {
+        this.snapshots = [];
+    }
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ * @internal
+ */
+export function evaluateLoadSignalSample(sample, overloadedRatio) {
+    if (sample.length === 0) {
+        return {
+            isOverloaded: false,
+            limitRatio: overloadedRatio,
+            actualRatio: 0,
+        };
+    }
+    const weights = [];
+    const values = [];
+    for (let i = 1; i < sample.length; i++) {
+        const previous = sample[i - 1];
+        const current = sample[i];
+        const weight = +current.createdAt - +previous.createdAt;
+        weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
+        values.push(+current.isOverloaded);
+    }
+    const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
+    return {
+        isOverloaded: wAvg > overloadedRatio,
+        limitRatio: overloadedRatio,
+        actualRatio: Math.round(wAvg * 1000) / 1000,
+    };
+}

package/autoscaling/memory_load_signal.d.ts

@@ -0,0 +1,56 @@
+import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
+import type { SystemInfo } from './system_status.js';
+/**
+ * A snapshot produced by the built-in memory signal.
+ * @internal
+ */
+export interface MemorySnapshot extends LoadSnapshot {
+    usedBytes?: number;
+}
+/**
+ * Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
+ * {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
+ */
+export interface MemoryLoadSignalOptions {
+    /**
+     * Defines the maximum ratio of total memory that can be used.
+     * Exceeding this limit overloads the memory.
+     * @default 0.9
+     */
+    maxUsedRatio?: number;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
+     * @default 0.2
+     */
+    overloadedRatio?: number;
+}
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
+ * threshold. Also warns when memory use becomes critical.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export declare class MemoryLoadSignal implements LoadSignal {
+    readonly name = "memInfo";
+    readonly overloadedRatio: number;
+    private readonly store;
+    private readonly maxUsedRatio;
+    /** All resolved in `start()`, before anything that reads them can fire. */
+    private config;
+    private log;
+    private maxMemoryBytes;
+    private events?;
+    private maxMemoryRatio;
+    private lastLoggedCriticalMemoryOverloadAt;
+    constructor(options?: MemoryLoadSignalOptions);
+    start(context: LoadSignalStartContext): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
+    handle(systemInfo: SystemInfo): void;
+    /** @internal */
+    _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
+    private _getTotalMemoryBytes;
+}

package/autoscaling/memory_load_signal.js

@@ -0,0 +1,106 @@
+import { serviceLocator } from '../service_locator.js';
+import { getMemoryInfo } from '../system-info/memory-info.js';
+import { isContainerized } from '../system-info/runtime.js';
+import { SnapshotStore } from './load_signal.js';
+const RESERVE_MEMORY_RATIO = 0.5;
+const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
+ * threshold. Also warns when memory use becomes critical.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export class MemoryLoadSignal {
+    name = 'memInfo';
+    overloadedRatio;
+    store = new SnapshotStore();
+    maxUsedRatio;
+    /** All resolved in `start()`, before anything that reads them can fire. */
+    config;
+    log;
+    maxMemoryBytes;
+    events;
+    maxMemoryRatio;
+    lastLoggedCriticalMemoryOverloadAt = null;
+    constructor(options = {}) {
+        this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
+        this.overloadedRatio = options.overloadedRatio ?? 0.2;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
+        this.store.clear();
+        // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
+        // between systems) must not capture whichever services happened to be registered at that moment.
+        this.config = serviceLocator.getConfiguration();
+        this.events = serviceLocator.getEventManager();
+        this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
+        const memoryMbytes = this.config.memoryMbytes ?? 0;
+        if (memoryMbytes > 0) {
+            this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
+        }
+        else {
+            this.maxMemoryRatio = this.config.availableMemoryRatio;
+            if (!this.maxMemoryRatio) {
+                throw new Error('availableMemoryRatio is not set in configuration.');
+            }
+            else {
+                this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
+                    'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
+            }
+            // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
+            this.maxMemoryBytes = await this._getTotalMemoryBytes();
+        }
+        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+    }
+    async stop() {
+        this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+        this.events = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.store.getSample(sampleDurationMillis);
+    }
+    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
+    handle(systemInfo) {
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        const { memCurrentBytes, memTotalBytes } = systemInfo;
+        let maxMemoryBytes = this.maxMemoryBytes;
+        if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
+            maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
+        }
+        const snapshot = {
+            createdAt,
+            isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
+            usedBytes: memCurrentBytes,
+        };
+        this.store.push(snapshot, createdAt);
+        this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
+    }
+    /** @internal */
+    _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
+        const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
+        const { memCurrentBytes } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        if (this.lastLoggedCriticalMemoryOverloadAt &&
+            +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+            return;
+        const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
+        const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
+        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
+        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
+        if (isCriticalOverload) {
+            const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
+            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
+            this.log.warning('Memory is critically overloaded. ' +
+                `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
+            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
+        }
+    }
+    async _getTotalMemoryBytes() {
+        const containerized = this.config.containerized ?? (await isContainerized());
+        return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
+    }
+}