@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/autoscaling/system_status.d.ts

@@ -0,0 +1,181 @@
+import { Snapshotter } from './snapshotter';
+import { Configuration } from '../configuration';
+/**
+ * Represents the current status of the system.
+ */
+export interface SystemInfo {
+    /** If true, system is being overloaded. */
+    isSystemIdle: boolean;
+    memInfo: ClientInfo;
+    eventLoopInfo: ClientInfo;
+    cpuInfo: ClientInfo;
+    clientInfo: ClientInfo;
+    memCurrentBytes?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    cpuCurrentUsage?: number;
+    /**
+     * Platform only property
+     * @internal
+     */
+    isCpuOverloaded?: boolean;
+    /**
+     * Platform only property
+     * @internal
+     */
+    createdAt?: Date;
+}
+export interface SystemStatusOptions {
+    /**
+     * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
+     * @default 5
+     */
+    currentHistorySecs?: number;
+    /**
+     * Sets the maximum ratio of overloaded snapshots in a memory sample.
+     * If the sample exceeds this ratio, the system will be overloaded.
+     * @default 0.2
+     */
+    maxMemoryOverloadedRatio?: number;
+    /**
+     * Sets the maximum ratio of overloaded snapshots in an event loop sample.
+     * If the sample exceeds this ratio, the system will be overloaded.
+     * @default 0.6
+     */
+    maxEventLoopOverloadedRatio?: number;
+    /**
+     * Sets the maximum ratio of overloaded snapshots in a CPU sample.
+     * If the sample exceeds this ratio, the system will be overloaded.
+     * @default 0.4
+     */
+    maxCpuOverloadedRatio?: number;
+    /**
+     * Sets the maximum ratio of overloaded snapshots in a Client sample.
+     * If the sample exceeds this ratio, the system will be overloaded.
+     * @default 0.3
+     */
+    maxClientOverloadedRatio?: number;
+    /**
+     * The `Snapshotter` instance to be queried for `SystemStatus`.
+     */
+    snapshotter?: Snapshotter;
+    /** @internal */
+    config?: Configuration;
+}
+export interface ClientInfo {
+    isOverloaded: boolean;
+    limitRatio: number;
+    actualRatio: number;
+}
+export interface FinalStatistics {
+    requestsFinished: number;
+    requestsFailed: number;
+    retryHistogram: number[];
+    requestAvgFailedDurationMillis: number;
+    requestAvgFinishedDurationMillis: number;
+    requestsFinishedPerMinute: number;
+    requestsFailedPerMinute: number;
+    requestTotalDurationMillis: number;
+    requestsTotal: number;
+    crawlerRuntimeMillis: number;
+}
+/**
+ * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
+ * It only exposes two functions {@link SystemStatus.getCurrentStatus}
+ * and {@link SystemStatus.getHistoricalStatus}.
+ * The system status is calculated using a weighted average of overloaded
+ * messages in the snapshots, with the weights being the time intervals
+ * between the snapshots. Each resource is calculated separately
+ * and the system is overloaded whenever at least one resource is overloaded.
+ * The class is used by the {@link AutoscaledPool} class.
+ *
+ * {@link SystemStatus.getCurrentStatus}
+ * returns a boolean that represents the current status of the system.
+ * The length of the current timeframe in seconds is configurable
+ * by the `currentHistorySecs` option and represents the max age
+ * of snapshots to be considered for the calculation.
+ *
+ * {@link SystemStatus.getHistoricalStatus}
+ * returns a boolean that represents the long-term status
+ * of the system. It considers the full snapshot history available
+ * in the {@link Snapshotter} instance.
+ * @category Scaling
+ */
+export declare class SystemStatus {
+    private readonly currentHistorySecs;
+    private readonly maxMemoryOverloadedRatio;
+    private readonly maxEventLoopOverloadedRatio;
+    private readonly maxCpuOverloadedRatio;
+    private readonly maxClientOverloadedRatio;
+    private readonly snapshotter;
+    constructor(options?: SystemStatusOptions);
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the last `options.currentHistorySecs` seconds,
+     * and `true` otherwise.
+     */
+    getCurrentStatus(): SystemInfo;
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the full history of the {@link Snapshotter}
+     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     */
+    getHistoricalStatus(): SystemInfo;
+    /**
+     * Returns a system status object.
+     */
+    protected _isSystemIdle(sampleDurationMillis?: number): SystemInfo;
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the memory has been overloaded in the last sampleDurationMillis.
+     */
+    protected _isMemoryOverloaded(sampleDurationMillis?: number): ClientInfo;
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the event loop has been overloaded in the last sampleDurationMillis.
+     */
+    protected _isEventLoopOverloaded(sampleDurationMillis?: number): ClientInfo;
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the CPU has been overloaded in the last sampleDurationMillis.
+     */
+    protected _isCpuOverloaded(sampleDurationMillis?: number): ClientInfo;
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the client has been overloaded in the last sampleDurationMillis.
+     */
+    protected _isClientOverloaded(sampleDurationMillis?: number): ClientInfo;
+    /**
+     * Returns an object with sample information and an isOverloaded property
+     * set to true if at least the ratio of snapshots in the sample are overloaded.
+     */
+    protected _isSampleOverloaded<T extends {
+        createdAt: Date;
+        isOverloaded: boolean;
+    }>(sample: T[], ratio: number): ClientInfo;
+}
+//# sourceMappingURL=system_status.d.ts.map

package/dist/autoscaling/system_status.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"system_status.d.ts","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,2CAA2C;IAC3C,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,UAAU,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,SAAS,CAAC,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAS;IACrD,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,GAAE,mBAAwB;IA6B7C;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB,IAAI,UAAU;IAI9B;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB,IAAI,UAAU;IAIjC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAclE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK3D;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAK9D;;;OAGG;IACH,SAAS,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAKxD;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,UAAU;IAKxE;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU;CAoB9H"}

package/dist/autoscaling/system_status.js

@@ -0,0 +1,194 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SystemStatus = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const utils_1 = require("@crawlee/utils");
+const snapshotter_1 = require("./snapshotter");
+/**
+ * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
+ * It only exposes two functions {@link SystemStatus.getCurrentStatus}
+ * and {@link SystemStatus.getHistoricalStatus}.
+ * The system status is calculated using a weighted average of overloaded
+ * messages in the snapshots, with the weights being the time intervals
+ * between the snapshots. Each resource is calculated separately
+ * and the system is overloaded whenever at least one resource is overloaded.
+ * The class is used by the {@link AutoscaledPool} class.
+ *
+ * {@link SystemStatus.getCurrentStatus}
+ * returns a boolean that represents the current status of the system.
+ * The length of the current timeframe in seconds is configurable
+ * by the `currentHistorySecs` option and represents the max age
+ * of snapshots to be considered for the calculation.
+ *
+ * {@link SystemStatus.getHistoricalStatus}
+ * returns a boolean that represents the long-term status
+ * of the system. It considers the full snapshot history available
+ * in the {@link Snapshotter} instance.
+ * @category Scaling
+ */
+class SystemStatus {
+    constructor(options = {}) {
+        Object.defineProperty(this, "currentHistorySecs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxMemoryOverloadedRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxEventLoopOverloadedRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxCpuOverloadedRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxClientOverloadedRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "snapshotter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            currentHistorySecs: ow_1.default.optional.number,
+            maxMemoryOverloadedRatio: ow_1.default.optional.number,
+            maxEventLoopOverloadedRatio: ow_1.default.optional.number,
+            maxCpuOverloadedRatio: ow_1.default.optional.number,
+            maxClientOverloadedRatio: ow_1.default.optional.number,
+            snapshotter: ow_1.default.optional.object,
+            config: ow_1.default.optional.object,
+        }));
+        const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, config, } = options;
+        this.currentHistorySecs = currentHistorySecs * 1000;
+        this.maxMemoryOverloadedRatio = maxMemoryOverloadedRatio;
+        this.maxEventLoopOverloadedRatio = maxEventLoopOverloadedRatio;
+        this.maxCpuOverloadedRatio = maxCpuOverloadedRatio;
+        this.maxClientOverloadedRatio = maxClientOverloadedRatio;
+        this.snapshotter = snapshotter || new snapshotter_1.Snapshotter({ config });
+    }
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the last `options.currentHistorySecs` seconds,
+     * and `true` otherwise.
+     */
+    getCurrentStatus() {
+        return this._isSystemIdle(this.currentHistorySecs);
+    }
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the full history of the {@link Snapshotter}
+     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     */
+    getHistoricalStatus() {
+        return this._isSystemIdle();
+    }
+    /**
+     * Returns a system status object.
+     */
+    _isSystemIdle(sampleDurationMillis) {
+        const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
+        const eventLoopInfo = this._isEventLoopOverloaded(sampleDurationMillis);
+        const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
+        const clientInfo = this._isClientOverloaded(sampleDurationMillis);
+        return {
+            isSystemIdle: !memInfo.isOverloaded && !eventLoopInfo.isOverloaded && !cpuInfo.isOverloaded && !clientInfo.isOverloaded,
+            memInfo,
+            eventLoopInfo,
+            cpuInfo,
+            clientInfo,
+        };
+    }
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the memory has been overloaded in the last sampleDurationMillis.
+     */
+    _isMemoryOverloaded(sampleDurationMillis) {
+        const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
+        return this._isSampleOverloaded(sample, this.maxMemoryOverloadedRatio);
+    }
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the event loop has been overloaded in the last sampleDurationMillis.
+     */
+    _isEventLoopOverloaded(sampleDurationMillis) {
+        const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
+        return this._isSampleOverloaded(sample, this.maxEventLoopOverloadedRatio);
+    }
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the CPU has been overloaded in the last sampleDurationMillis.
+     */
+    _isCpuOverloaded(sampleDurationMillis) {
+        const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
+        return this._isSampleOverloaded(sample, this.maxCpuOverloadedRatio);
+    }
+    /**
+     * Returns an object with an isOverloaded property set to true
+     * if the client has been overloaded in the last sampleDurationMillis.
+     */
+    _isClientOverloaded(sampleDurationMillis) {
+        const sample = this.snapshotter.getClientSample(sampleDurationMillis);
+        return this._isSampleOverloaded(sample, this.maxClientOverloadedRatio);
+    }
+    /**
+     * Returns an object with sample information and an isOverloaded property
+     * set to true if at least the ratio of snapshots in the sample are overloaded.
+     */
+    _isSampleOverloaded(sample, ratio) {
+        const weights = [];
+        const values = [];
+        for (let i = 1; i < sample.length; i++) {
+            const previous = sample[i - 1];
+            const current = sample[i];
+            const weight = +current.createdAt - +previous.createdAt;
+            weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
+            values.push(+current.isOverloaded);
+        }
+        const wAvg = (0, utils_1.weightedAvg)(values, weights);
+        return {
+            isOverloaded: wAvg > ratio,
+            limitRatio: ratio,
+            actualRatio: Math.round(wAvg * 1000) / 1000,
+        };
+    }
+}
+exports.SystemStatus = SystemStatus;
+//# sourceMappingURL=system_status.js.map

package/dist/autoscaling/system_status.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"system_status.js","sourceRoot":"","sources":["../../src/autoscaling/system_status.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,0CAA6C;AAC7C,+CAA4C;AA8F5C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAa,YAAY;IAQrB,YAAY,UAA+B,EAAE;QAP7C;;;;;WAA4C;QAC5C;;;;;WAAkD;QAClD;;;;;WAAqD;QACrD;;;;;WAA+C;QAC/C;;;;;WAAkD;QAClD;;;;;WAA0C;QAGtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACzC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5C,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,kBAAkB,GAAG,CAAC,EACtB,wBAAwB,GAAG,GAAG,EAC9B,2BAA2B,GAAG,GAAG,EACjC,qBAAqB,GAAG,GAAG,EAC3B,wBAAwB,GAAG,GAAG,EAC9B,WAAW,EACX,MAAM,GACT,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QACzD,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,WAAW,IAAI,IAAI,yBAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAClE,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,gBAAgB;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACvD,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,mBAAmB;QACf,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,oBAA6B;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;QAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;QAClE,OAAO;YACH,YAAY,EAAE,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,YAAY;YACvH,OAAO;YACP,aAAa;YACb,OAAO;YACP,UAAU;SACb,CAAC;IACN,CAAC;IAED;;;OAGG;IACO,mBAAmB,CAAC,oBAA6B;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACO,sBAAsB,CAAC,oBAA6B;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC9E,CAAC;IAED;;;OAGG;IACO,gBAAgB,CAAC,oBAA6B;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACxE,CAAC;IAED;;;OAGG;IACO,mBAAmB,CAAC,oBAA6B;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACO,mBAAmB,CAAuD,MAAW,EAAE,KAAa;QAC1G,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,mEAAmE;YAC9F,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;SACtC;QAED,MAAM,IAAI,GAAG,IAAA,mBAAW,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE1C,OAAO;YACH,YAAY,EAAE,IAAI,GAAG,KAAK;YAC1B,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;SAC9C,CAAC;IACN,CAAC;CACJ;AA1JD,oCA0JC"}

package/dist/cache_container.d.ts

@@ -0,0 +1,18 @@
+import { LruCache } from '@apify/datastructures';
+/**
+ * Used to manage all globally created caches, such as request queue cache
+ * or dataset cache. Before creation of this class, those caches were
+ * created as module scoped globals - untouchable. This proved problematic
+ * especially in tests, where caches would prevent test separation.
+ * @ignore
+ */
+export declare class CacheContainer {
+    caches: Map<string, LruCache<any>>;
+    openCache(name: string, maxSize: number): LruCache;
+    getCache(name: string): LruCache | undefined;
+    clearCache(name: string): void;
+    clearAllCaches(): void;
+}
+declare const _default: CacheContainer;
+export default _default;
+//# sourceMappingURL=cache_container.d.ts.map

package/dist/cache_container.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache_container.d.ts","sourceRoot":"","sources":["../src/cache_container.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD;;;;;;GAMG;AACH,qBAAa,cAAc;IACvB,MAAM,6BAA+B;IAErC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ;IASlD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAI5C,UAAU,CAAC,IAAI,EAAE,MAAM;IAKvB,cAAc;CAGjB;;AAED,wBAAoC"}

package/dist/cache_container.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CacheContainer = void 0;
+const datastructures_1 = require("@apify/datastructures");
+/**
+ * Used to manage all globally created caches, such as request queue cache
+ * or dataset cache. Before creation of this class, those caches were
+ * created as module scoped globals - untouchable. This proved problematic
+ * especially in tests, where caches would prevent test separation.
+ * @ignore
+ */
+class CacheContainer {
+    constructor() {
+        Object.defineProperty(this, "caches", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+    }
+    openCache(name, maxSize) {
+        let cache = this.caches.get(name);
+        if (!cache) {
+            cache = new datastructures_1.LruCache({ maxLength: maxSize });
+            this.caches.set(name, cache);
+        }
+        return cache;
+    }
+    getCache(name) {
+        return this.caches.get(name);
+    }
+    clearCache(name) {
+        const cache = this.caches.get(name);
+        cache?.clear();
+    }
+    clearAllCaches() {
+        this.caches.forEach((cache) => cache.clear());
+    }
+}
+exports.CacheContainer = CacheContainer;
+exports.default = new CacheContainer();
+//# sourceMappingURL=cache_container.js.map

package/dist/cache_container.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache_container.js","sourceRoot":"","sources":["../src/cache_container.ts"],"names":[],"mappings":";;;AAAA,0DAAiD;AAEjD;;;;;;GAMG;AACH,MAAa,cAAc;IAA3B;QACI;;;;mBAAS,IAAI,GAAG,EAAoB;WAAC;IAuBzC,CAAC;IArBG,SAAS,CAAC,IAAY,EAAE,OAAe;QACnC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE;YACR,KAAK,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAChC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,QAAQ,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,UAAU,CAAC,IAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,KAAK,EAAE,KAAK,EAAE,CAAC;IACnB,CAAC;IAED,cAAc;QACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAClD,CAAC;CACJ;AAxBD,wCAwBC;AAED,kBAAe,IAAI,cAAc,EAAE,CAAC"}

package/dist/configuration.d.ts

@@ -0,0 +1,150 @@
+/// <reference types="node" />
+import { ApifyStorageLocal, ApifyStorageLocalOptions } from '@apify/storage-local';
+import { AsyncLocalStorage } from 'async_hooks';
+import { Dictionary } from './typedefs';
+import { StorageClient } from './storages/storage';
+import { EventManager } from './events';
+export interface ConfigurationOptions {
+    storageClient?: StorageClient;
+    eventManager?: EventManager;
+    storageClientOptions?: Dictionary;
+    defaultDatasetId?: string;
+    defaultKeyValueStoreId?: string;
+    defaultRequestQueueId?: string;
+    maxUsedCpuRatio?: number;
+    availableMemoryRatio?: number;
+    persistStateIntervalMillis?: number;
+    systemInfoIntervalMillis?: number;
+    metamorphAfterSleepMillis?: number;
+    inputKey?: string;
+    actorId?: string;
+    actorRunId?: string;
+    actorTaskId?: string;
+    containerPort?: number;
+    containerUrl?: string;
+    proxyHostname?: string;
+    proxyPassword?: string;
+    proxyPort?: number;
+}
+/**
+ * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
+ *
+ * 1. When using `Apify` class, we can get the instance configuration via `sdk.config`
+ *     ```js
+ *     const { Apify } = require('apify');
+ *
+ *     const sdk = new Apify({ token: '123' });
+ *     console.log(sdk.config.get('token')); // '123'
+ *     ```
+ * 2. To get the global configuration (singleton instance). It will respect the environment variables.
+ *     ```js
+ *     // returns the token from APIFY_TOKEN env var
+ *     console.log(Configuration.getGlobalConfig().get('token'));
+ *     ```
+ *
+ * ## Supported Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
+ * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
+ * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
+ * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
+ *
+ * ## Advanced Configuration Options
+ *
+ * Key | Environment Variable | Default Value
+ * ---|---|---
+ * `actorId` | `APIFY_ACTOR_ID` | -
+ * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
+ * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
+ * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
+ * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
+ * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
+ * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
+ * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
+ * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
+ * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
+ *
+ * ## Not Supported environment variables
+ *
+ * - `MEMORY_MBYTES`
+ * - `HEADLESS`
+ * - `XVFB`
+ * - `CHROME_EXECUTABLE_PATH`
+ */
+export declare class Configuration {
+    /**
+     * Maps environment variables to config keys (e.g. `APIFY_PROXY_PORT` to `proxyPort`)
+     */
+    private static ENV_MAP;
+    /**
+     * Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
+     */
+    private static ENV_MAP_REVERSED;
+    private static BOOLEAN_VARS;
+    private static INTEGER_VARS;
+    private static DEFAULTS;
+    /**
+     * Provides access to the current-instance-scoped Configuration without passing it around in parameters.
+     * @internal
+     */
+    static storage: AsyncLocalStorage<Configuration>;
+    private options;
+    private services;
+    private static globalConfig?;
+    /**
+     * Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
+     */
+    constructor(options?: ConfigurationOptions);
+    /**
+     * Returns configured value. First checks the environment variables, then provided configuration,
+     * fallbacks to the `defaultValue` argument if provided, otherwise uses the default value as described
+     * in the above section.
+     */
+    get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
+    private _castEnvValue;
+    /**
+     * Sets value for given option. Only affects this `Configuration` instance, the value will not be propagated down to the env var.
+     * To reset a value, we can omit the `value` argument or pass `undefined` there.
+     */
+    set(key: keyof ConfigurationOptions, value?: any): void;
+    /**
+     * Returns cached instance of {@link StorageClient} using options as defined in the environment variables or in
+     * this {@link Configuration} instance. Only first call of this method will create the client, following calls will
+     * return the same client instance.
+     *
+     * Caching works based on the API URL and token, so calling this method with different options will return
+     * multiple instances, one for each variant of the options.
+     * @internal
+     */
+    getStorageClient(): StorageClient;
+    getEventManager(): EventManager;
+    /**
+     * Creates an instance of ApifyStorageLocal using options as defined in the environment variables or in this `Configuration` instance.
+     * @internal
+     */
+    createStorageLocal(options?: ApifyStorageLocalOptions): ApifyStorageLocal;
+    useStorageClient(client: StorageClient): void;
+    useEventManager(events: EventManager): void;
+    /**
+     * Returns the global configuration instance. It will respect the environment variables.
+     * When used inside SDK instance (`sdk.main()`), the return value is equal to `sdk.config`.
+     *
+     */
+    static getGlobalConfig(): Configuration;
+    /**
+     * Gets default {@link StorageClient} instance.
+     */
+    static getStorageClient(): StorageClient;
+    /**
+     * Gets default {@link EventManager} instance.
+     */
+    static getEventManager(): EventManager;
+    /**
+     * Resets global configuration instance. The default instance holds configuration based on env vars,
+     * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
+     */
+    static resetGlobalState(): void;
+}
+//# sourceMappingURL=configuration.d.ts.map

package/dist/configuration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAW,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAG3D,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,OAAO,CAyBpB;IAEF;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAGE;IAEjC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAgB;IAE3C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAkD;IAE7E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAerB;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,OAAO,CAAC,OAAO,CAAoF;IACnG,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAgB;IAE5C;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAO9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IAkBzG,OAAO,CAAC,aAAa;IAarB;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,kBAAkB,CAAC,OAAO,GAAE,wBAA6B,GAAG,iBAAiB;IAkB7E,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;;;OAIG;IACH,MAAM,CAAC,eAAe,IAAK,aAAa;IASxC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;CAGlC"}