@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/autoscaling/snapshotter.d.ts

@@ -0,0 +1,193 @@
+import { Log } from '@apify/log';
+import { BetterIntervalID } from '@apify/utilities';
+import { Configuration } from '../configuration';
+import { SystemInfo } from './system_status';
+import { StorageClient } from '../storages/storage';
+import { EventManager } from '../events/event_manager';
+export interface SnapshotterOptions {
+    /**
+     * Defines the interval of measuring the event loop response time.
+     * @default 0.5
+     */
+    eventLoopSnapshotIntervalSecs?: number;
+    /**
+     * Defines the interval of checking the current state
+     * of the remote API client.
+     * @default 1
+     */
+    clientSnapshotIntervalSecs?: number;
+    /**
+     * Maximum allowed delay of the event loop in milliseconds.
+     * Exceeding this limit overloads the event loop.
+     * @default 50
+     */
+    maxBlockedMillis?: number;
+    /**
+     * Defines the maximum ratio of total memory that can be used.
+     * Exceeding this limit overloads the memory.
+     * @default 0.7
+     */
+    maxUsedMemoryRatio?: number;
+    /**
+     * Defines the maximum number of new rate limit errors within
+     * the given interval.
+     * @default 1
+     */
+    maxClientErrors?: number;
+    /**
+     * Sets the interval in seconds for which a history of resource snapshots
+     * will be kept. Increasing this to very high numbers will affect performance.
+     * @default 60
+     */
+    snapshotHistorySecs?: number;
+    /** @internal */
+    log?: Log;
+    /** @internal */
+    client?: StorageClient;
+    /** @internal */
+    config?: Configuration;
+}
+declare type MemorySnapshot = {
+    createdAt: Date;
+    isOverloaded: boolean;
+    usedBytes?: number;
+};
+declare type CpuSnapshot = {
+    createdAt: Date;
+    isOverloaded: boolean;
+    usedRatio: number;
+    ticks?: {
+        idle: number;
+        total: number;
+    };
+};
+declare type EventLoopSnapshot = {
+    createdAt: Date;
+    isOverloaded: boolean;
+    exceededMillis: number;
+};
+declare type ClientSnapshot = {
+    createdAt: Date;
+    isOverloaded: boolean;
+    rateLimitErrorCount: number;
+};
+/**
+ * Creates snapshots of system resources at given intervals and marks the resource
+ * as either overloaded or not during the last interval. Keeps a history of the snapshots.
+ * It tracks the following resources: Memory, EventLoop, API and CPU.
+ * The class is used by the {@link AutoscaledPool} class.
+ *
+ * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
+ * as collected from the running Docker container. When running locally, `Snapshotter`
+ * makes its own statistics by querying the OS.
+ *
+ * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
+ * when Apify platform marks it as overloaded.
+ *
+ * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
+ * It's computed using the total memory available to the container when running on
+ * the Apify platform and a quarter of total system memory when running locally.
+ * Max total memory when running locally may be overridden by using the `APIFY_MEMORY_MBYTES`
+ * environment variable.
+ *
+ * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
+ *
+ * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
+ * typically received from the request queue, exceed the set limit within the set interval.
+ * @category Scaling
+ */
+export declare class Snapshotter {
+    log: Log;
+    client: StorageClient;
+    config: Configuration;
+    events: EventManager;
+    eventLoopSnapshotIntervalMillis: number;
+    clientSnapshotIntervalMillis: number;
+    snapshotHistoryMillis: number;
+    maxBlockedMillis: number;
+    maxUsedMemoryRatio: number;
+    maxClientErrors: number;
+    maxMemoryBytes: number;
+    cpuSnapshots: CpuSnapshot[];
+    eventLoopSnapshots: EventLoopSnapshot[];
+    memorySnapshots: MemorySnapshot[];
+    clientSnapshots: ClientSnapshot[];
+    eventLoopInterval: BetterIntervalID;
+    clientInterval: BetterIntervalID;
+    lastLoggedCriticalMemoryOverloadAt: Date | null;
+    /**
+     * @param [options] All `Snapshotter` configuration options.
+     */
+    constructor(options?: SnapshotterOptions);
+    /**
+     * Starts capturing snapshots at configured intervals.
+     */
+    start(): Promise<void>;
+    /**
+     * Stops all resource capturing.
+     */
+    stop(): Promise<void>;
+    /**
+     * Returns a sample of latest memory snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];
+    /**
+     * Returns a sample of latest event loop snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getEventLoopSample(sampleDurationMillis?: number): EventLoopSnapshot[];
+    /**
+     * Returns a sample of latest CPU snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getCpuSample(sampleDurationMillis?: number): CpuSnapshot[];
+    /**
+     * Returns a sample of latest Client snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
+    /**
+     * Finds the latest snapshots by sampleDurationMillis in the provided array.
+     */
+    protected _getSample<T extends {
+        createdAt: Date;
+    }>(snapshots: T[], sampleDurationMillis?: number): T[];
+    /**
+     * Creates a snapshot of current memory usage
+     * using the Apify platform `systemInfo` event.
+     */
+    protected _snapshotMemory(systemInfo: SystemInfo): void;
+    /**
+     * Checks for critical memory overload and logs it to the console.
+     */
+    protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
+    /**
+     * Creates a snapshot of current event loop delay.
+     */
+    protected _snapshotEventLoop(intervalCallback: () => unknown): void;
+    /**
+     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
+     */
+    protected _snapshotCpu(systemInfo: SystemInfo): void;
+    /**
+     * Creates a snapshot of current API state by checking for
+     * rate limit errors. Only errors produced by a 2nd retry
+     * of the API call are considered for snapshotting since
+     * earlier errors may just be caused by a random spike in
+     * number of requests and do not necessarily signify API
+     * overloading.
+     */
+    protected _snapshotClient(intervalCallback: () => unknown): void;
+    /**
+     * Removes snapshots that are older than the snapshotHistorySecs option
+     * from the array (destructively - in place).
+     */
+    protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
+    /**
+     * Helper method for easier mocking.
+     */
+    private _getMemoryInfo;
+}
+export {};
+//# sourceMappingURL=snapshotter.d.ts.map

package/dist/autoscaling/snapshotter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAG5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAa,MAAM,yBAAyB,CAAC;AAMlE,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,aAAK,cAAc,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AACrF,aAAK,WAAW,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAC1H,aAAK,iBAAiB,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC;AAC5F,aAAK,cAAc,GAAG;IAAE,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,mBAAmB,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9F;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA0C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAe5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAU3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IAmBvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EAAE,GAAG,EAAE,IAAI;IAUzH;;OAEG;YACW,cAAc;CAG/B"}

package/dist/autoscaling/snapshotter.js

@@ -0,0 +1,377 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Snapshotter = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const utils_1 = require("@crawlee/utils");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const configuration_1 = require("../configuration");
+const log_1 = require("../log");
+const RESERVE_MEMORY_RATIO = 0.5;
+const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
+/**
+ * Creates snapshots of system resources at given intervals and marks the resource
+ * as either overloaded or not during the last interval. Keeps a history of the snapshots.
+ * It tracks the following resources: Memory, EventLoop, API and CPU.
+ * The class is used by the {@link AutoscaledPool} class.
+ *
+ * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
+ * as collected from the running Docker container. When running locally, `Snapshotter`
+ * makes its own statistics by querying the OS.
+ *
+ * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
+ * when Apify platform marks it as overloaded.
+ *
+ * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
+ * It's computed using the total memory available to the container when running on
+ * the Apify platform and a quarter of total system memory when running locally.
+ * Max total memory when running locally may be overridden by using the `APIFY_MEMORY_MBYTES`
+ * environment variable.
+ *
+ * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
+ *
+ * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
+ * typically received from the request queue, exceed the set limit within the set interval.
+ * @category Scaling
+ */
+class Snapshotter {
+    /**
+     * @param [options] All `Snapshotter` configuration options.
+     */
+    constructor(options = {}) {
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "eventLoopSnapshotIntervalMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "clientSnapshotIntervalMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "snapshotHistoryMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxBlockedMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxUsedMemoryRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxClientErrors", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxMemoryBytes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cpuSnapshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "eventLoopSnapshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "memorySnapshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "clientSnapshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "eventLoopInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "clientInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "lastLoggedCriticalMemoryOverloadAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            eventLoopSnapshotIntervalSecs: ow_1.default.optional.number,
+            clientSnapshotIntervalSecs: ow_1.default.optional.number,
+            snapshotHistorySecs: ow_1.default.optional.number,
+            maxBlockedMillis: ow_1.default.optional.number,
+            maxUsedMemoryRatio: ow_1.default.optional.number,
+            maxClientErrors: ow_1.default.optional.number,
+            log: ow_1.default.optional.object,
+            client: ow_1.default.optional.object,
+            config: ow_1.default.optional.object,
+        }));
+        const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.7, maxClientErrors = 3, log = log_1.log, config = configuration_1.Configuration.getGlobalConfig(), client = config.getStorageClient(), } = options;
+        this.log = log.child({ prefix: 'Snapshotter' });
+        this.client = client;
+        this.config = config;
+        this.events = this.config.getEventManager();
+        this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
+        this.clientSnapshotIntervalMillis = clientSnapshotIntervalSecs * 1000;
+        this.snapshotHistoryMillis = snapshotHistorySecs * 1000;
+        this.maxBlockedMillis = maxBlockedMillis;
+        this.maxUsedMemoryRatio = maxUsedMemoryRatio;
+        this.maxClientErrors = maxClientErrors;
+        // We need to pre-bind those functions to be able to successfully remove listeners.
+        this._snapshotCpu = this._snapshotCpu.bind(this);
+        this._snapshotMemory = this._snapshotMemory.bind(this);
+    }
+    /**
+     * Starts capturing snapshots at configured intervals.
+     */
+    async start() {
+        const { totalBytes } = await this._getMemoryInfo();
+        this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
+        this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the AVAILABLE_MEMORY_RATIO environment variable to override it.`); // eslint-disable-line max-len
+        // Ensure events are initialized.
+        // await this.events.init();
+        // Start snapshotting.
+        this.eventLoopInterval = (0, utilities_1.betterSetInterval)(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
+        this.clientInterval = (0, utilities_1.betterSetInterval)(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
+        this.events.on("systemInfo" /* SYSTEM_INFO */, this._snapshotCpu);
+        this.events.on("systemInfo" /* SYSTEM_INFO */, this._snapshotMemory);
+    }
+    /**
+     * Stops all resource capturing.
+     */
+    async stop() {
+        (0, utilities_1.betterClearInterval)(this.eventLoopInterval);
+        (0, utilities_1.betterClearInterval)(this.clientInterval);
+        this.events.off("systemInfo" /* SYSTEM_INFO */, this._snapshotCpu);
+        this.events.off("systemInfo" /* SYSTEM_INFO */, this._snapshotMemory);
+        // Allow microtask queue to unwind before stop returns.
+        await new Promise((resolve) => setImmediate(resolve));
+        // await this.events.close();
+    }
+    /**
+     * Returns a sample of latest memory snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getMemorySample(sampleDurationMillis) {
+        return this._getSample(this.memorySnapshots, sampleDurationMillis);
+    }
+    /**
+     * Returns a sample of latest event loop snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getEventLoopSample(sampleDurationMillis) {
+        return this._getSample(this.eventLoopSnapshots, sampleDurationMillis);
+    }
+    /**
+     * Returns a sample of latest CPU snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getCpuSample(sampleDurationMillis) {
+        return this._getSample(this.cpuSnapshots, sampleDurationMillis);
+    }
+    /**
+     * Returns a sample of latest Client snapshots, with the size of the sample defined
+     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
+     */
+    getClientSample(sampleDurationMillis) {
+        return this._getSample(this.clientSnapshots, sampleDurationMillis);
+    }
+    /**
+     * Finds the latest snapshots by sampleDurationMillis in the provided array.
+     */
+    _getSample(snapshots, sampleDurationMillis) {
+        if (!sampleDurationMillis)
+            return snapshots;
+        const sample = [];
+        let idx = snapshots.length;
+        if (!idx)
+            return sample;
+        const latestTime = snapshots[idx - 1].createdAt;
+        while (idx--) {
+            const snapshot = snapshots[idx];
+            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
+                sample.unshift(snapshot);
+            }
+            else {
+                break;
+            }
+        }
+        return sample;
+    }
+    /**
+     * Creates a snapshot of current memory usage
+     * using the Apify platform `systemInfo` event.
+     */
+    _snapshotMemory(systemInfo) {
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        this._pruneSnapshots(this.memorySnapshots, createdAt);
+        const { memCurrentBytes } = systemInfo;
+        const snapshot = {
+            createdAt,
+            isOverloaded: memCurrentBytes / this.maxMemoryBytes > this.maxUsedMemoryRatio,
+            usedBytes: memCurrentBytes,
+        };
+        this.memorySnapshots.push(snapshot);
+        this._memoryOverloadWarning(systemInfo);
+    }
+    /**
+     * Checks for critical memory overload and logs it to the console.
+     */
+    _memoryOverloadWarning(systemInfo) {
+        const { memCurrentBytes } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        if (this.lastLoggedCriticalMemoryOverloadAt && +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+            return;
+        const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes;
+        const reserveMemory = this.maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
+        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
+        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
+        if (isCriticalOverload) {
+            const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
+            const toMb = (bytes) => Math.round(bytes / (1024 ** 2));
+            this.log.warning('Memory is critically overloaded. '
+                + `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing the actor memory.`);
+            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
+        }
+    }
+    /**
+     * Creates a snapshot of current event loop delay.
+     */
+    _snapshotEventLoop(intervalCallback) {
+        const now = new Date();
+        this._pruneSnapshots(this.eventLoopSnapshots, now);
+        const snapshot = {
+            createdAt: now,
+            isOverloaded: false,
+            exceededMillis: 0,
+        };
+        const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];
+        if (previousSnapshot) {
+            const { createdAt } = previousSnapshot;
+            const delta = now.getTime() - +createdAt - this.eventLoopSnapshotIntervalMillis;
+            if (delta > this.maxBlockedMillis)
+                snapshot.isOverloaded = true;
+            snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);
+        }
+        this.eventLoopSnapshots.push(snapshot);
+        intervalCallback();
+    }
+    /**
+     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
+     */
+    _snapshotCpu(systemInfo) {
+        const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        this._pruneSnapshots(this.cpuSnapshots, createdAt);
+        this.cpuSnapshots.push({
+            createdAt,
+            isOverloaded: isCpuOverloaded,
+            usedRatio: Math.ceil(cpuCurrentUsage / 100),
+        });
+    }
+    /**
+     * Creates a snapshot of current API state by checking for
+     * rate limit errors. Only errors produced by a 2nd retry
+     * of the API call are considered for snapshotting since
+     * earlier errors may just be caused by a random spike in
+     * number of requests and do not necessarily signify API
+     * overloading.
+     */
+    _snapshotClient(intervalCallback) {
+        const now = new Date();
+        this._pruneSnapshots(this.clientSnapshots, now);
+        const allErrorCounts = this.client.stats?.rateLimitErrors ?? []; // storage client might not support this
+        const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+        // Handle empty snapshots array
+        const snapshot = {
+            createdAt: now,
+            isOverloaded: false,
+            rateLimitErrorCount: currentErrCount,
+        };
+        const previousSnapshot = this.clientSnapshots[this.clientSnapshots.length - 1];
+        if (previousSnapshot) {
+            const { rateLimitErrorCount } = previousSnapshot;
+            const delta = currentErrCount - rateLimitErrorCount;
+            if (delta > this.maxClientErrors)
+                snapshot.isOverloaded = true;
+        }
+        this.clientSnapshots.push(snapshot);
+        intervalCallback();
+    }
+    /**
+     * Removes snapshots that are older than the snapshotHistorySecs option
+     * from the array (destructively - in place).
+     */
+    _pruneSnapshots(snapshots, now) {
+        let oldCount = 0;
+        for (let i = 0; i < snapshots.length; i++) {
+            const { createdAt } = snapshots[i];
+            if (now.getTime() - new Date(createdAt).getTime() > this.snapshotHistoryMillis)
+                oldCount++;
+            else
+                break;
+        }
+        snapshots.splice(0, oldCount);
+    }
+    /**
+     * Helper method for easier mocking.
+     */
+    async _getMemoryInfo() {
+        return (0, utils_1.getMemoryInfo)();
+    }
+}
+exports.Snapshotter = Snapshotter;
+//# sourceMappingURL=snapshotter.js.map

package/dist/autoscaling/snapshotter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":";;;;AACA,gDAA4F;AAC5F,0CAA+C;AAC/C,oDAAoB;AACpB,oDAAiD;AACjD,gCAA2C;AAK3C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA2DlD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAa,WAAW;IAuBpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAzB5C;;;;;WAAS;QACT;;;;;WAAsB;QACtB;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAwC;QACxC;;;;;WAAqC;QACrC;;;;;WAA8B;QAC9B;;;;;WAAyB;QACzB;;;;;WAA2B;QAC3B;;;;;WAAwB;QACxB;;;;;WAAwB;QAExB;;;;mBAA8B,EAAE;WAAC;QACjC;;;;mBAA0C,EAAE;WAAC;QAC7C;;;;mBAAoC,EAAE;WAAC;QACvC;;;;mBAAoC,EAAE;WAAC;QAEvC;;;;mBAAsC,IAAK;WAAC;QAC5C;;;;mBAAmC,IAAK;WAAC;QAEzC;;;;mBAAkD,IAAI;WAAC;QAMnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,6BAA6B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,SAAU,EAChB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;QACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,0EAA0E,CAAC,CAAC,CAAC,8BAA8B;QAE5M,iCAAiC;QACjC,4BAA4B;QAE5B,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACrH,IAAI,CAAC,cAAc,GAAG,IAAA,6BAAiB,EAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,iCAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,IAAA,+BAAmB,EAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,IAAA,+BAAmB,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,iCAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,iCAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QACtD,6BAA6B;IACjC,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE;gBAC3D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC5B;iBAAM;gBACH,MAAM;aACT;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,IAAI,CAAC,kCAAkC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAAE,OAAO;QAEnJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE;YACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,mCAAmC;kBAC9C,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,cAAe,CAAC,QAAQ,cAAc,2CAA2C,CAAC,CAAC;YAC5I,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;SACvD;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;SACxE;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE;YAClB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;SAClE;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,SAAoF,EAAE,GAAS;QACrH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;SACd;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,OAAO,IAAA,qBAAa,GAAE,CAAC;IAC3B,CAAC;CACJ;AA/RD,kCA+RC"}