apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/autoscaling/snapshotter.d.ts

@@ -1,278 +0,0 @@
-export default Snapshotter;
-export type SnapshotterOptions = {
-    /**
-     * Defines the interval of measuring the event loop response time.
-     */
-    eventLoopSnapshotIntervalSecs?: number | undefined;
-    /**
-     * Defines the interval of checking the current state
-     * of the remote API client.
-     */
-    clientSnapshotIntervalSecs?: number | undefined;
-    /**
-     * Maximum allowed delay of the event loop in milliseconds.
-     * Exceeding this limit overloads the event loop.
-     */
-    maxBlockedMillis?: number | undefined;
-    /**
-     * Defines the interval of measuring CPU usage.
-     * This is only used when running locally. On the Apify platform,
-     * the statistics are provided externally at a fixed interval.
-     */
-    cpuSnapshotIntervalSecs?: number | undefined;
-    /**
-     * Defines the maximum usage of CPU.
-     * Exceeding this limit overloads the CPU.
-     */
-    maxUsedCpuRatio?: number | undefined;
-    /**
-     * Defines the interval of measuring memory consumption.
-     * This is only used when running locally. On the Apify platform,
-     * the statistics are provided externally at a fixed interval.
-     * The measurement itself is resource intensive (25 - 50ms async).
-     * Therefore, setting this interval below 1 second is not recommended.
-     */
-    memorySnapshotIntervalSecs?: number | undefined;
-    /**
-     * Defines the maximum ratio of total memory that can be used.
-     * Exceeding this limit overloads the memory.
-     */
-    maxUsedMemoryRatio?: number | undefined;
-    /**
-     * Defines the maximum number of new rate limit errors within
-     * the given interval.
-     */
-    maxClientErrors?: number | undefined;
-    /**
-     * Sets the interval in seconds for which a history of resource snapshots
-     * will be kept. Increasing this to very high numbers will affect performance.
-     */
-    snapshotHistorySecs?: number | undefined;
-};
-/**
- * @typedef SnapshotterOptions
- * @property {number} [eventLoopSnapshotIntervalSecs=0.5]
- *   Defines the interval of measuring the event loop response time.
- * @property {number} [clientSnapshotIntervalSecs=1]
- *   Defines the interval of checking the current state
- *   of the remote API client.
- * @property {number} [maxBlockedMillis=50]
- *   Maximum allowed delay of the event loop in milliseconds.
- *   Exceeding this limit overloads the event loop.
- * @property {number} [cpuSnapshotIntervalSecs=1]
- *   Defines the interval of measuring CPU usage.
- *   This is only used when running locally. On the Apify platform,
- *   the statistics are provided externally at a fixed interval.
- * @property {number} [maxUsedCpuRatio=0.95]
- *   Defines the maximum usage of CPU.
- *   Exceeding this limit overloads the CPU.
- * @property {number} [memorySnapshotIntervalSecs=1]
- *   Defines the interval of measuring memory consumption.
- *   This is only used when running locally. On the Apify platform,
- *   the statistics are provided externally at a fixed interval.
- *   The measurement itself is resource intensive (25 - 50ms async).
- *   Therefore, setting this interval below 1 second is not recommended.
- * @property {number} [maxUsedMemoryRatio=0.7]
- *   Defines the maximum ratio of total memory that can be used.
- *   Exceeding this limit overloads the memory.
- * @property {number} [maxClientErrors=1]
- *   Defines the maximum number of new rate limit errors within
- *   the given interval.
- * @property {number} [snapshotHistorySecs=60]
- *   Sets the interval in seconds for which a history of resource snapshots
- *   will be kept. Increasing this to very high numbers will affect performance.
- */
-/**
- * Creates snapshots of system resources at given intervals and marks the resource
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
- * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@link AutoscaledPool} class.
- *
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
- * as collected from the running Docker container. When running locally, `Snapshotter`
- * makes its own statistics by querying the OS.
- *
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
- * when Apify platform marks it as overloaded.
- *
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
- * It's computed using the total memory available to the container when running on
- * the Apify platform and a quarter of total system memory when running locally.
- * Max total memory when running locally may be overridden by using the `APIFY_MEMORY_MBYTES`
- * environment variable.
- *
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
- *
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
- * typically received from the request queue, exceed the set limit within the set interval.
- */
-declare class Snapshotter {
-    /**
-     * @param {SnapshotterOptions} [options] All `Snapshotter` configuration options.
-     */
-    constructor(options?: SnapshotterOptions | undefined);
-    /** @type {Log} */
-    log: Log;
-    eventLoopSnapshotIntervalMillis: number;
-    memorySnapshotIntervalMillis: number;
-    clientSnapshotIntervalMillis: number;
-    cpuSnapshotIntervalMillis: number;
-    snapshotHistoryMillis: number;
-    maxBlockedMillis: number;
-    maxUsedMemoryRatio: number;
-    maxUsedCpuRatio: number;
-    maxClientErrors: number;
-    maxMemoryBytes: number | null;
-    cpuSnapshots: any[];
-    eventLoopSnapshots: any[];
-    memorySnapshots: any[];
-    clientSnapshots: any[];
-    eventLoopInterval: import("@apify/utilities").BetterIntervalID | null;
-    memoryInterval: import("@apify/utilities").BetterIntervalID | null;
-    clientInterval: import("@apify/utilities").BetterIntervalID | null;
-    cpuInterval: import("@apify/utilities").BetterIntervalID | null;
-    lastLoggedCriticalMemoryOverloadAt: Date | null;
-    /**
-     * Creates a snapshot of current CPU usage
-     * using the Apify platform `systemInfo` event.
-     * @param {Object} systemInfo
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotCpuOnPlatform(systemInfo: Object): void;
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
-     * @param {*} systemInfo
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotMemoryOnPlatform(systemInfo: any): void;
-    /**
-     * Starts capturing snapshots at configured intervals.
-     * @return {Promise<void>}
-     */
-    start(): Promise<void>;
-    /**
-     * Stops all resource capturing.
-     * @return {Promise<void>}
-     */
-    stop(): Promise<void>;
-    /**
-     * Returns a sample of latest memory snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     * @param {number} [sampleDurationMillis]
-     * @return {Array<*>}
-     */
-    getMemorySample(sampleDurationMillis?: number | undefined): Array<any>;
-    /**
-     * Returns a sample of latest event loop snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     * @param {number} [sampleDurationMillis]
-     * @return {Array<*>}
-     */
-    getEventLoopSample(sampleDurationMillis?: number | undefined): Array<any>;
-    /**
-     * Returns a sample of latest CPU snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     * @param {number} [sampleDurationMillis]
-     * @return {Array<*>}
-     */
-    getCpuSample(sampleDurationMillis?: number | undefined): Array<any>;
-    /**
-     * Returns a sample of latest Client snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     * @param {number} sampleDurationMillis
-     * @return {Array<*>}
-     */
-    getClientSample(sampleDurationMillis: number): Array<any>;
-    /**
-     * Finds the latest snapshots by sampleDurationMillis in the provided array.
-     * @param {Array<*>} snapshots
-     * @param {number} [sampleDurationMillis]
-     * @return {Array<*>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _getSample(snapshots: Array<any>, sampleDurationMillis?: number | undefined): Array<any>;
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
-     * @param {Function} intervalCallback
-     * @return {Promise<void>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotMemoryOnLocal(intervalCallback: Function): Promise<void>;
-    /**
-     * Checks for critical memory overload and logs it to the console.
-     * @param {*} systemInfo
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _memoryOverloadWarning({ memCurrentBytes }: any): void;
-    /**
-     * Creates a snapshot of current event loop delay.
-     * @param {Function} intervalCallback
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotEventLoop(intervalCallback: Function): void;
-    /**
-     * Creates a snapshot of current CPU usage
-     * using OS provided metrics.
-     * @param {Function} intervalCallback
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotCpuOnLocal(intervalCallback: Function): void;
-    /**
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _getCurrentCpuTicks(): {
-        idle: number;
-        total: number;
-    };
-    /**
-     * Creates a snapshot of current API state by checking for
-     * rate limit errors. Only errors produced by a 2nd retry
-     * of the API call are considered for snapshotting since
-     * earlier errors may just be caused by a random spike in
-     * number of requests and do not necessarily signify API
-     * overloading.
-     *
-     * @param intervalCallback
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _snapshotClient(intervalCallback: any): void;
-    /**
-     * Removes snapshots that are older than the snapshotHistorySecs option
-     * from the array (destructively - in place).
-     * @param {Array<*>} snapshots
-     * @param {Date} now
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _pruneSnapshots(snapshots: Array<any>, now: Date): void;
-    /**
-     * Calculate max memory for platform or local usage.
-     * @ignore
-     * @protected
-     * @internal
-     */
-    protected _ensureCorrectMaxMemory(): Promise<void>;
-}
-import { Log } from "../utils_log";
-//# sourceMappingURL=snapshotter.d.ts.map

package/build/autoscaling/snapshotter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAYA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH;IACI;;OAEG;IACH,sDAwDC;IA7BG,kBAAkB;IAClB,KADW,GAAG,CACiC;IAE/C,wCAA2E;IAC3E,qCAAqE;IACrE,qCAAqE;IACrE,kCAA+D;IAC/D,8BAAuD;IACvD,yBAAwC;IACxC,2BAA4C;IAC5C,wBAAsC;IACtC,wBAAsC;IACtC,8BAA+F;IAE/F,oBAAsB;IACtB,0BAA4B;IAC5B,uBAAyB;IACzB,uBAAyB;IAEzB,sEAA6B;IAC7B,mEAA0B;IAC1B,mEAA0B;IAC1B,gEAAuB;IAEvB,gDAA8C;IAsNlD;;;;;;;OAOG;IACH,6CALW,MAAM,QAehB;IA9HD;;;;;;;OAOG;IACH,2DAYC;IAvHD;;;OAGG;IACH,SAFY,QAAQ,IAAI,CAAC,CAexB;IAED;;;OAGG;IACH,QAFY,QAAQ,IAAI,CAAC,CAWxB;IAED;;;;;OAKG;IACH,4DAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,+DAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,yDAFY,UAAQ,CAInB;IAED;;;;;OAKG;IACH,sCAHW,MAAM,GACL,UAAQ,CAInB;IAED;;;;;;;;OAQG;IACH,gCAPW,UAAQ,8CAEP,UAAQ,CAmBnB;IAwBD;;;;;;;;OAQG;IACH,8DALY,QAAQ,IAAI,CAAC,CAyBxB;IAED;;;;;;OAMG;IACH,iEAgBC;IAED;;;;;;OAMG;IACH,+DAqBC;IAsBD;;;;;;;OAOG;IACH,gEAyBC;IAED;;;;OAIG;IACH;;;MASC;IAED;;;;;;;;;;;;OAYG;IACH,uDAsBC;IAED;;;;;;;;OAQG;IACH,qCANW,UAAQ,OACR,IAAI,QAad;IAED;;;;;OAKG;IACH,mDAUC;CACJ"}