@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.110

package/autoscaling/snapshotter.js

@@ -1,285 +1,67 @@
-import { getMemoryInfo, isContainerized } from '@crawlee/utils';
-import ow from 'ow';
-import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
-import { log as defaultLog } from '../log.js';
-const RESERVE_MEMORY_RATIO = 0.5;
-const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
-const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
+import { ClientLoadSignal } from './client_load_signal.js';
+import { CpuLoadSignal } from './cpu_load_signal.js';
+import { EventLoopLoadSignal } from './event_loop_load_signal.js';
+import { MemoryLoadSignal } from './memory_load_signal.js';
 /**
- * Creates snapshots of system resources at given intervals and marks the resource
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
- * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@link AutoscaledPool} class.
+ * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
+ * that were not switched off and driving their shared lifecycle.
  *
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
- * as collected from the running Docker container. When running locally, `Snapshotter`
- * makes its own statistics by querying the OS.
- *
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
- * when Apify platform marks it as overloaded.
- *
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
- * It's computed using the total memory available to the container when running on
- * the Apify platform and a quarter of total system memory when running locally.
- * Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
- * environment variable.
- *
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
- *
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
- * typically received from the request queue, exceed the set limit within the set interval.
- * @category Scaling
+ * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
+ * are simply forwarded to the corresponding constructor.
+ * @internal
  */
 export class Snapshotter {
-    log;
-    client;
-    config;
-    events;
-    eventLoopSnapshotIntervalMillis;
-    clientSnapshotIntervalMillis;
-    snapshotHistoryMillis;
-    maxBlockedMillis;
-    maxUsedMemoryRatio;
-    maxClientErrors;
-    maxMemoryBytes;
-    cpuSnapshots = [];
-    eventLoopSnapshots = [];
-    memorySnapshots = [];
-    clientSnapshots = [];
-    eventLoopInterval = null;
-    clientInterval = null;
-    lastLoggedCriticalMemoryOverloadAt = null;
+    // Absent when switched off through the corresponding option (e.g. `client: false`).
+    #memorySignal;
+    #eventLoopSignal;
+    #cpuSignal;
+    #clientSignal;
+    /**
+     * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
+     * instances. Signals switched off through the options are simply absent — the system status reports them as
+     * not overloaded.
+     */
+    getLoadSignals() {
+        const builtin = [
+            this.#memorySignal,
+            this.#eventLoopSignal,
+            this.#cpuSignal,
+            this.#clientSignal,
+        ];
+        return builtin.filter((signal) => signal !== undefined);
+    }
     /**
      * @param [options] All `Snapshotter` configuration options.
      */
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            eventLoopSnapshotIntervalSecs: ow.optional.number,
-            clientSnapshotIntervalSecs: ow.optional.number,
-            snapshotHistorySecs: ow.optional.number,
-            maxBlockedMillis: ow.optional.number,
-            maxUsedMemoryRatio: ow.optional.number,
-            maxClientErrors: ow.optional.number,
-            log: ow.optional.object,
-            client: ow.optional.object,
-            config: ow.optional.object,
-        }));
-        const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = defaultLog, config = Configuration.getGlobalConfig(), client = config.getStorageClient(), } = options;
-        this.log = log.child({ prefix: 'Snapshotter' });
-        this.client = client;
-        this.config = config;
-        this.events = this.config.getEventManager();
-        this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
-        this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
-        this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
-        this.maxBlockedMillis = maxBlockedMillis;
-        this.maxUsedMemoryRatio = maxUsedMemoryRatio;
-        this.maxClientErrors = maxClientErrors;
-        // We need to pre-bind those functions to be able to successfully remove listeners.
-        this._snapshotCpu = this._snapshotCpu.bind(this);
-        this._snapshotMemory = this._snapshotMemory.bind(this);
+        const { memory = {}, eventLoop = {}, cpu = {}, client = {} } = options;
+        // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
+        // over then too - so there is nothing to thread in here beyond the caller's tuning.
+        if (memory !== false)
+            this.#memorySignal = new MemoryLoadSignal(memory);
+        if (eventLoop !== false)
+            this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
+        if (cpu !== false)
+            this.#cpuSignal = new CpuLoadSignal(cpu);
+        if (client !== false)
+            this.#clientSignal = new ClientLoadSignal(client);
     }
     /**
-     * Starts capturing snapshots at configured intervals.
+     * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
+     * be queried with, which is also how much history they retain.
      */
-    async start() {
-        const memoryMbytes = this.config.get('memoryMbytes', 0);
-        if (memoryMbytes > 0) {
-            this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
-        }
-        else {
-            const containerized = this.config.get('containerized', await isContainerized());
-            const memInfo = await getMemoryInfo(containerized);
-            const totalBytes = memInfo.totalBytes;
-            this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
-            this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
-                'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
-        }
-        // Start snapshotting.
-        this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
-        this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
-        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
-        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
+    async start(context) {
+        await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
     }
     /**
      * Stops all resource capturing.
      */
     async stop() {
-        betterClearInterval(this.eventLoopInterval);
-        betterClearInterval(this.clientInterval);
-        this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotCpu);
-        this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._snapshotMemory);
+        await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
         // Allow microtask queue to unwind before stop returns.
         await new Promise((resolve) => {
             setImmediate(resolve);
         });
     }
-    /**
-     * Returns a sample of latest memory snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getMemorySample(sampleDurationMillis) {
-        return this._getSample(this.memorySnapshots, sampleDurationMillis);
-    }
-    /**
-     * Returns a sample of latest event loop snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getEventLoopSample(sampleDurationMillis) {
-        return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
-    }
-    /**
-     * Returns a sample of latest CPU snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getCpuSample(sampleDurationMillis) {
-        return this._getSample(this.cpuSnapshots, sampleDurationMillis);
-    }
-    /**
-     * Returns a sample of latest Client snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getClientSample(sampleDurationMillis) {
-        return this._getSample(this.clientSnapshots, sampleDurationMillis);
-    }
-    /**
-     * Finds the latest snapshots by sampleDurationMillis in the provided array.
-     */
-    _getSample(snapshots, sampleDurationMillis) {
-        if (!sampleDurationMillis)
-            return snapshots;
-        const sample = [];
-        let idx = snapshots.length;
-        if (!idx)
-            return sample;
-        const latestTime = snapshots[idx - 1].createdAt;
-        while (idx--) {
-            const snapshot = snapshots[idx];
-            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
-                sample.unshift(snapshot);
-            }
-            else {
-                break;
-            }
-        }
-        return sample;
-    }
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
-     */
-    _snapshotMemory(systemInfo) {
-        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        this._pruneSnapshots(this.memorySnapshots, createdAt);
-        const { memCurrentBytes } = systemInfo;
-        const snapshot = {
-            createdAt,
-            isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
-            usedBytes: memCurrentBytes,
-        };
-        this.memorySnapshots.push(snapshot);
-        this._memoryOverloadWarning(systemInfo);
-    }
-    /**
-     * Checks for critical memory overload and logs it to the console.
-     */
-    _memoryOverloadWarning(systemInfo) {
-        const { memCurrentBytes } = systemInfo;
-        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        if (this.lastLoggedCriticalMemoryOverloadAt &&
-            +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
-            return;
-        const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
-        const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
-        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
-        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
-        if (isCriticalOverload) {
-            const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
-            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
-            this.log.warning('Memory is critically overloaded. ' +
-                `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing available memory.`);
-            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
-        }
-    }
-    /**
-     * Creates a snapshot of current event loop delay.
-     */
-    _snapshotEventLoop(intervalCallback) {
-        const now = new Date();
-        this._pruneSnapshots(this.eventLoopSnapshots, now);
-        const snapshot = {
-            createdAt: now,
-            isOverloaded: false,
-            exceededMillis: 0,
-        };
-        const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
-        if (previousSnapshot) {
-            const { createdAt } = previousSnapshot;
-            const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
-            if (delta > this.maxBlockedMillis)
-                snapshot.isOverloaded = true;
-            snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
-        }
-        this.eventLoopSnapshots.push(snapshot);
-        intervalCallback();
-    }
-    /**
-     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
-     */
-    _snapshotCpu(systemInfo) {
-        const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
-        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        this._pruneSnapshots(this.cpuSnapshots, createdAt);
-        this.cpuSnapshots.push({
-            createdAt,
-            isOverloaded: isCpuOverloaded,
-            usedRatio: Math.ceil(cpuCurrentUsage / 100),
-        });
-    }
-    /**
-     * Creates a snapshot of current API state by checking for
-     * rate limit errors. Only errors produced by a 2nd retry
-     * of the API call are considered for snapshotting since
-     * earlier errors may just be caused by a random spike in
-     * number of requests and do not necessarily signify API
-     * overloading.
-     */
-    _snapshotClient(intervalCallback) {
-        const now = new Date();
-        this._pruneSnapshots(this.clientSnapshots, now);
-        const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
-        const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
-        // Handle empty snapshots array
-        const snapshot = {
-            createdAt: now,
-            isOverloaded: false,
-            rateLimitErrorCount: currentErrCount,
-        };
-        const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
-        if (previousSnapshot) {
-            const { rateLimitErrorCount } = previousSnapshot;
-            const delta = currentErrCount - rateLimitErrorCount;
-            if (delta > this.maxClientErrors)
-                snapshot.isOverloaded = true;
-        }
-        this.clientSnapshots.push(snapshot);
-        intervalCallback();
-    }
-    /**
-     * Removes snapshots that are older than the snapshotHistorySecs option
-     * from the array (destructively - in place).
-     */
-    _pruneSnapshots(snapshots, now) {
-        let oldCount = 0;
-        for (let i = 0; i < snapshots.length; i++) {
-            const { createdAt } = snapshots[i];
-            if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
-                oldCount++;
-            else
-                break;
-        }
-        snapshots.splice(0, oldCount);
-    }
 }
-//# sourceMappingURL=snapshotter.js.map

package/autoscaling/system_status.d.ts

@@ -1,5 +1,5 @@
-import type { Configuration } from '../configuration.js';
-import { Snapshotter } from './snapshotter.js';
+import type { LoadSignal } from './load_signal.js';
+import type { Snapshotter } from './snapshotter.js';
 /**
  * Represents the current status of the system.
  */
@@ -10,6 +10,7 @@ export interface SystemInfo {
     eventLoopInfo: ClientInfo;
     cpuInfo: ClientInfo;
     clientInfo: ClientInfo;
+    memTotalBytes?: number;
     memCurrentBytes?: number;
     /**
      * Platform only property
@@ -26,7 +27,28 @@ export interface SystemInfo {
      * @internal
      */
     createdAt?: Date;
+    /**
+     * Status of additional load signals beyond the built-in four.
+     * Keys are `LoadSignal.name` values, values are overload info.
+     */
+    loadSignalInfo?: Record<string, ClientInfo>;
 }
+/**
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/**
+ * An implementation detail of the {@link ConcurrencySystem} — configure it through
+ * {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
+ * @internal
+ */
 export interface SystemStatusOptions {
     /**
      * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
@@ -34,35 +56,23 @@ export interface SystemStatusOptions {
      */
     currentHistorySecs?: number;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a memory sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.2
-     */
-    maxMemoryOverloadedRatio?: number;
-    /**
-     * Sets the maximum ratio of overloaded snapshots in an event loop sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.6
+     * Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
+     * autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
+     * private retention cannot silently widen the window.
+     * @default 30
      */
-    maxEventLoopOverloadedRatio?: number;
+    historySecs?: number;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a CPU sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.4
+     * The `Snapshotter` whose built-in signals are evaluated.
      */
-    maxCpuOverloadedRatio?: number;
+    snapshotter: Snapshotter;
     /**
-     * Sets the maximum ratio of overloaded snapshots in a Client sample.
-     * If the sample exceeds this ratio, the system will be overloaded.
-     * @default 0.3
+     * Additional load signals to include in the system status evaluation.
+     * These are evaluated alongside the built-in memory, CPU, event loop,
+     * and client signals. If any signal reports overload, the system is
+     * considered overloaded. Each signal carries its own overload ratio.
      */
-    maxClientOverloadedRatio?: number;
-    /**
-     * The `Snapshotter` instance to be queried for `SystemStatus`.
-     */
-    snapshotter?: Snapshotter;
-    /** @internal */
-    config?: Configuration;
+    loadSignals?: LoadSignal[];
 }
 export interface ClientInfo {
     isOverloaded: boolean;
@@ -82,35 +92,33 @@ export interface FinalStatistics {
     crawlerRuntimeMillis: number;
 }
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
- * The system status is calculated using a weighted average of overloaded
- * messages in the snapshots, with the weights being the time intervals
- * between the snapshots. Each resource is calculated separately
- * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
  *
- * {@link SystemStatus.getCurrentStatus}
- * returns a boolean that represents the current status of the system.
- * The length of the current timeframe in seconds is configurable
- * by the `currentHistorySecs` option and represents the max age
- * of snapshots to be considered for the calculation.
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
  *
- * {@link SystemStatus.getHistoricalStatus}
- * returns a boolean that represents the long-term status
- * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
- * @category Scaling
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
  */
 export declare class SystemStatus {
-    private readonly currentHistoryMillis;
-    private readonly maxMemoryOverloadedRatio;
-    private readonly maxEventLoopOverloadedRatio;
-    private readonly maxCpuOverloadedRatio;
-    private readonly maxClientOverloadedRatio;
-    private readonly snapshotter;
-    constructor(options?: SystemStatusOptions);
+    #private;
+    constructor(options: SystemStatusOptions);
+    /**
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
+     */
+    get maxSampleWindowMillis(): number;
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    private assertUniqueSignalNames;
     /**
      * Returns an {@link SystemInfo} object with the following structure:
      *
@@ -140,42 +148,12 @@ export declare class SystemStatus {
      * }
      * ```
      *
-     * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
      */
     getHistoricalStatus(): SystemInfo;
     /**
      * Returns a system status object.
      */
-    protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the memory has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the event loop has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the CPU has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with an isOverloaded property set to true
-     * if the client has been overloaded in the last sampleDurationMillis.
-     */
-    protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
-    /**
-     * Returns an object with sample information and an isOverloaded property
-     * set to true if at least the ratio of snapshots in the sample are overloaded.
-     */
-    protected _isSampleOverloaded<T extends {
-        createdAt: Date;
-        isOverloaded: boolean;
-    }>(sample: T[], ratio: number): ClientInfo;
+    private isSystemIdle;
 }
-//# sourceMappingURL=system_status.d.ts.map