@crawlee/core 3.16.1-beta.33 → 3.16.1-beta.35

package/autoscaling/load_signal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load_signal.js","sourceRoot":"","sources":["../../src/autoscaling/load_signal.ts"],"names":[],"mappings":";;;AA2LA,4DA2BC;AAtND,0CAA6C;AAG7C,gDAA0E;AA6C1E;;;GAGG;AACH,MAAa,aAAa;IAItB,YAAY,aAAa,GAAG,KAAM;QAH1B;;;;mBAAiB,EAAE;WAAC;QACX;;;;;WAAsB;QAGnC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,QAAW,EAAE,MAAY,QAAQ,CAAC,SAAS;QAC5C,8DAA8D;QAC9D,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,aAAa;gBAAE,QAAQ,EAAE,CAAC;;gBAC9E,MAAM;QACf,CAAC;QACD,IAAI,QAAQ;YAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAEjD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,oBAA6B;QACnC,IAAI,CAAC,oBAAoB;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAEjD,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QACrD,OAAO,GAAG,EAAE,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,MAAM;YACV,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,YAAY,CAAyB,OAM3C;QAKG,MAAM,KAAK,GAAG,IAAI,aAAa,CAAI,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAClE,IAAI,QAAQ,GAAqB,IAAK,CAAC;QAEvC,MAAM,MAAM,GAAG,CAAC,EAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAEjE,OAAO;YACH,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK;YACL,MAAM;YACN,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,KAAK,CAAC,KAAK;gBACP,QAAQ,GAAG,IAAA,6BAAiB,EAAC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;YACjE,CAAC;YACD,KAAK,CAAC,IAAI;gBACN,IAAA,+BAAmB,EAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;SACJ,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,SAAS,CAA4B,OAO3C;QAKG,MAAM,KAAK,GAAG,IAAI,aAAa,CAAI,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAElE,MAAM,MAAM,GAAG,CAAC,OAAU,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/D,OAAO;YACH,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK;YACL,MAAM;YACN,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,KAAK,CAAC,KAAK;gBACP,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC7C,CAAC;YACD,KAAK,CAAC,IAAI;gBACN,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AAhID,sCAgIC;AAED;;;;GAIG;AACH,SAAgB,wBAAwB,CAAC,MAAsB,EAAE,eAAuB;IACpF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO;YACH,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,eAAe;YAC3B,WAAW,EAAE,CAAC;SACjB,CAAC;IACN,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,mEAAmE;QAC9F,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAA,mBAAW,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAE1F,OAAO;QACH,YAAY,EAAE,IAAI,GAAG,eAAe;QACpC,UAAU,EAAE,eAAe;QAC3B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;KAC9C,CAAC;AACN,CAAC"}

package/autoscaling/memory_load_signal.d.ts

@@ -0,0 +1,44 @@
+import type { Log } from '@apify/log';
+import type { Configuration } from '../configuration';
+import type { LoadSignal, LoadSnapshot } from './load_signal';
+import type { SystemInfo } from './system_status';
+export interface MemorySnapshot extends LoadSnapshot {
+    usedBytes?: number;
+}
+export interface MemoryLoadSignalOptions {
+    maxUsedMemoryRatio?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+    config: Configuration;
+    log?: Log;
+}
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
+ * the used-to-available memory ratio exceeds a threshold.
+ */
+export declare class MemoryLoadSignal implements LoadSignal {
+    readonly name = "memInfo";
+    readonly overloadedRatio: number;
+    private readonly store;
+    private readonly config;
+    private readonly events;
+    private readonly log;
+    private readonly maxUsedMemoryRatio;
+    private maxMemoryRatio;
+    private maxMemoryBytes;
+    private lastLoggedCriticalMemoryOverloadAt;
+    constructor(options: MemoryLoadSignalOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): MemorySnapshot[];
+    /**
+     * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
+     */
+    getMemorySnapshots(): MemorySnapshot[];
+    /** @internal */
+    _onSystemInfo(systemInfo: SystemInfo): void;
+    /** @internal */
+    _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes?: number): void;
+    private _getTotalMemoryBytes;
+}
+//# sourceMappingURL=memory_load_signal.d.ts.map

package/autoscaling/memory_load_signal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory_load_signal.d.ts","sourceRoot":"","sources":["../../src/autoscaling/memory_load_signal.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAItD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAKlD,MAAM,WAAW,cAAe,SAAQ,YAAY;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,uBAAuB;IACpC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,MAAM,EAAE,aAAa,CAAC;IACtB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,UAAU;IAC/C,QAAQ,CAAC,IAAI,aAAa;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,kCAAkC,CAAqB;gBAEnD,OAAO,EAAE,uBAAuB;IAUtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,SAAS,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE;IAI1D;;OAEG;IACH,kBAAkB,IAAI,cAAc,EAAE;IAItC,gBAAgB;IAChB,aAAa,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI;IAmB3C,gBAAgB;IAChB,sBAAsB,CAAC,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;YA4B/D,oBAAoB;CAOrC"}

package/autoscaling/memory_load_signal.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryLoadSignal = void 0;
+const utils_1 = require("@crawlee/utils");
+const log_1 = require("../log");
+const load_signal_1 = require("./load_signal");
+const RESERVE_MEMORY_RATIO = 0.5;
+const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when
+ * the used-to-available memory ratio exceeds a threshold.
+ */
+class MemoryLoadSignal {
+    constructor(options) {
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 'memInfo'
+        });
+        Object.defineProperty(this, "overloadedRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "store", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxUsedMemoryRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxMemoryRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxMemoryBytes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "lastLoggedCriticalMemoryOverloadAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this.store = new load_signal_1.SnapshotStore(options.snapshotHistoryMillis);
+        this.config = options.config;
+        this.events = this.config.getEventManager();
+        this.log = options.log ?? log_1.log.child({ prefix: 'MemoryLoadSignal' });
+        this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
+        this.overloadedRatio = options.overloadedRatio ?? 0.2;
+        this._onSystemInfo = this._onSystemInfo.bind(this);
+    }
+    async start() {
+        const memoryMbytes = this.config.get('memoryMbytes', 0);
+        if (memoryMbytes > 0) {
+            this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
+        }
+        else {
+            this.maxMemoryRatio = this.config.get('availableMemoryRatio');
+            if (!this.maxMemoryRatio) {
+                throw new Error('availableMemoryRatio is not set in configuration.');
+            }
+            else {
+                this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
+                    'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
+            }
+            // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
+            this.maxMemoryBytes = await this._getTotalMemoryBytes();
+        }
+        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
+    }
+    async stop() {
+        this.events.off("systemInfo" /* EventType.SYSTEM_INFO */, this._onSystemInfo);
+    }
+    getSample(sampleDurationMillis) {
+        return this.store.getSample(sampleDurationMillis);
+    }
+    /**
+     * Returns typed memory snapshots for backward compatibility with `Snapshotter`.
+     */
+    getMemorySnapshots() {
+        return this.store.getAll();
+    }
+    /** @internal */
+    _onSystemInfo(systemInfo) {
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        const { memCurrentBytes, memTotalBytes } = systemInfo;
+        let maxMemoryBytes = this.maxMemoryBytes;
+        if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
+            maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
+        }
+        const snapshot = {
+            createdAt,
+            isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedMemoryRatio,
+            usedBytes: memCurrentBytes,
+        };
+        this.store.push(snapshot, createdAt);
+        this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
+    }
+    /** @internal */
+    _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
+        const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
+        const { memCurrentBytes } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        if (this.lastLoggedCriticalMemoryOverloadAt &&
+            +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+            return;
+        const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * effectiveMax;
+        const reserveMemory = effectiveMax * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
+        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
+        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
+        if (isCriticalOverload) {
+            const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
+            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
+            this.log.warning('Memory is critically overloaded. ' +
+                `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
+            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
+        }
+    }
+    async _getTotalMemoryBytes() {
+        if (this.config.get('systemInfoV2')) {
+            const containerized = this.config.get('containerized', await (0, utils_1.isContainerized)());
+            return (await (0, utils_1.getMemoryInfoV2)(containerized)).totalBytes;
+        }
+        return (await (0, utils_1.getMemoryInfo)()).totalBytes;
+    }
+}
+exports.MemoryLoadSignal = MemoryLoadSignal;
+//# sourceMappingURL=memory_load_signal.js.map

package/autoscaling/memory_load_signal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory_load_signal.js","sourceRoot":"","sources":["../../src/autoscaling/memory_load_signal.ts"],"names":[],"mappings":";;;AAAA,0CAAiF;AAOjF,gCAA2C;AAE3C,+CAA8C;AAG9C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,KAAM,CAAC;AAcnD;;;GAGG;AACH,MAAa,gBAAgB;IAazB,YAAY,OAAgC;QAZnC;;;;mBAAO,SAAS;WAAC;QACjB;;;;;WAAwB;QAEhB;;;;;WAAqC;QACrC;;;;;WAAsB;QACtB;;;;;WAAqB;QACrB;;;;;WAAS;QACT;;;;;WAA2B;QACpC;;;;;WAAmC;QACnC;;;;;WAAwB;QACxB;;;;mBAAkD,IAAI;WAAC;QAG3D,IAAI,CAAC,KAAK,GAAG,IAAI,2BAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAC5C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,GAAG,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YAC9D,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YACzE,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,qCAAqC,IAAI,CAAC,cAAc,GAAG,GAAG,0BAA0B;oBACpF,sGAAsG,CAC7G,CAAC;YACN,CAAC;YACD,gFAAgF;YAChF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,aAAa,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAI;QACN,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,aAAa,CAAC,CAAC;IAC/D,CAAC;IAED,SAAS,CAAC,oBAA6B;QACnC,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,kBAAkB;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/B,CAAC;IAED,gBAAgB;IAChB,aAAa,CAAC,UAAsB;QAChC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,MAAM,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,UAAU,CAAC;QAEtD,IAAI,cAAc,GAAG,IAAI,CAAC,cAAe,CAAC;QAC1C,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS,IAAI,IAAI,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC/D,cAAc,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,cAAc,GAAG,IAAI,CAAC,kBAAkB;YACzE,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,sBAAsB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC;IAED,gBAAgB;IAChB,sBAAsB,CAAC,UAAsB,EAAE,cAAuB;QAClE,MAAM,YAAY,GAAG,cAAc,IAAI,IAAI,CAAC,cAAe,CAAC;QAC5D,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IACI,IAAI,CAAC,kCAAkC;YACvC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAE3F,OAAO;QAEX,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,YAAY,CAAC;QACrE,MAAM,aAAa,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAC1F,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,mCAAmC;gBAC/B,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CACzC,YAAY,CACf,QAAQ,cAAc,2CAA2C,CACzE,CAAC;YACF,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;QACxD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,oBAAoB;QAC9B,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,IAAA,uBAAe,GAAE,CAAC,CAAC;YAChF,OAAO,CAAC,MAAM,IAAA,uBAAe,EAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC;QAC7D,CAAC;QACD,OAAO,CAAC,MAAM,IAAA,qBAAa,GAAE,CAAC,CAAC,UAAU,CAAC;IAC9C,CAAC;CACJ;AApHD,4CAoHC"}

package/autoscaling/snapshotter.d.ts

@@ -1,8 +1,12 @@
 import type { StorageClient } from '@crawlee/types';
 import type { Log } from '@apify/log';
-import type { BetterIntervalID } from '@apify/utilities';
 import { Configuration } from '../configuration';
 import type { EventManager } from '../events/event_manager';
+import type { ClientSnapshot } from './client_load_signal';
+import type { CpuSnapshot } from './cpu_load_signal';
+import type { EventLoopSnapshot } from './event_loop_load_signal';
+import type { LoadSignal } from './load_signal';
+import type { MemorySnapshot } from './memory_load_signal';
 import type { SystemInfo } from './system_status';
 export interface SnapshotterOptions {
     /**
@@ -47,30 +51,6 @@ export interface SnapshotterOptions {
     /** @internal */
     config?: Configuration;
 }
-interface MemorySnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedBytes?: number;
-}
-interface CpuSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedRatio: number;
-    ticks?: {
-        idle: number;
-        total: number;
-    };
-}
-interface EventLoopSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    exceededMillis: number;
-}
-interface ClientSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    rateLimitErrorCount: number;
-}
 /**
  * Creates snapshots of system resources at given intervals and marks the resource
  * as either overloaded or not during the last interval. Keeps a history of the snapshots.
@@ -94,6 +74,7 @@ interface ClientSnapshot {
  *
  * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
  * typically received from the request queue, exceed the set limit within the set interval.
+ *
  * @category Scaling
  */
 export declare class Snapshotter {
@@ -101,21 +82,19 @@ export declare class Snapshotter {
     client: StorageClient;
     config: Configuration;
     events: EventManager;
-    eventLoopSnapshotIntervalMillis: number;
-    clientSnapshotIntervalMillis: number;
-    snapshotHistoryMillis: number;
-    maxBlockedMillis: number;
-    maxUsedMemoryRatio: number;
-    maxClientErrors: number;
-    maxMemoryBytes: number;
-    private maxMemoryRatio;
-    cpuSnapshots: CpuSnapshot[];
-    eventLoopSnapshots: EventLoopSnapshot[];
-    memorySnapshots: MemorySnapshot[];
-    clientSnapshots: ClientSnapshot[];
-    eventLoopInterval: BetterIntervalID;
-    clientInterval: BetterIntervalID;
-    lastLoggedCriticalMemoryOverloadAt: Date | null;
+    private readonly memorySignal;
+    private readonly eventLoopSignal;
+    private readonly cpuSignal;
+    private readonly clientSignal;
+    /**
+     * Returns the four built-in signals as an array, so `SystemStatus` can
+     * iterate them alongside any custom `LoadSignal` instances.
+     */
+    getLoadSignals(): LoadSignal[];
+    get cpuSnapshots(): CpuSnapshot[];
+    get eventLoopSnapshots(): EventLoopSnapshot[];
+    get memorySnapshots(): MemorySnapshot[];
+    get clientSnapshots(): ClientSnapshot[];
     /**
      * @param [options] All `Snapshotter` configuration options.
      */
@@ -149,43 +128,28 @@ export declare class Snapshotter {
      */
     getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
     /**
-     * Finds the latest snapshots by sampleDurationMillis in the provided array.
-     */
-    protected _getSample<T extends {
-        createdAt: Date;
-    }>(snapshots: T[], sampleDurationMillis?: number): T[];
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
+     * @deprecated Kept for backward compatibility.
      */
     protected _snapshotMemory(systemInfo: SystemInfo): void;
     /**
-     * Checks for critical memory overload and logs it to the console.
+     * @deprecated Kept for backward compatibility.
      */
-    protected _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes: number): void;
+    protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
     /**
-     * Creates a snapshot of current event loop delay.
+     * @deprecated Kept for backward compatibility.
      */
     protected _snapshotEventLoop(intervalCallback: () => unknown): void;
     /**
-     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
+     * @deprecated Kept for backward compatibility.
      */
     protected _snapshotCpu(systemInfo: SystemInfo): void;
     /**
-     * Creates a snapshot of current API state by checking for
-     * rate limit errors. Only errors produced by a 2nd retry
-     * of the API call are considered for snapshotting since
-     * earlier errors may just be caused by a random spike in
-     * number of requests and do not necessarily signify API
-     * overloading.
+     * @deprecated Kept for backward compatibility.
      */
     protected _snapshotClient(intervalCallback: () => unknown): void;
     /**
-     * Removes snapshots that are older than the snapshotHistorySecs option
-     * from the array (destructively - in place).
+     * @deprecated Pruning is now handled by individual signals.
      */
-    protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
-    protected getTotalMemoryBytes(): Promise<number>;
+    protected _pruneSnapshots(_snapshots: any[], _now: Date): void;
 }
-export {};
 //# sourceMappingURL=snapshotter.d.ts.map

package/autoscaling/snapshotter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAMlD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IACxB,OAAO,CAAC,cAAc,CAAqB;IAE3C,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA4C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgC5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAoBhD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM;IA2B/E;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EACpF,GAAG,EAAE,IAAI;cAWG,mBAAmB;CAOtC"}
+{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,KAAK,EAAoB,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE7E,OAAO,KAAK,EAAiB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEpE,OAAO,KAAK,EAAuB,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAEvF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IAErB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAEhD;;;OAGG;IACH,cAAc,IAAI,UAAU,EAAE;IAK9B,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,kBAAkB,IAAI,iBAAiB,EAAE,CAE5C;IAED,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED,IAAI,eAAe,IAAI,cAAc,EAAE,CAEtC;IAED;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6D5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE;IAIhE;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAItE;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,WAAW,EAAE;IAI1D;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE;IAIhE;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAIhD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IAIvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAI5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAI7C;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAIzD;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI;CAG1D"}