@crawlee/basic 4.0.0-beta.19 → 4.0.0-beta.190

package/internals/autoscaling/system_status.d.ts

@@ -0,0 +1,122 @@
+import type { SystemInfo } from '@crawlee/core';
+import type { LoadSignal } from './load_signal.js';
+import type { Snapshotter } from './snapshotter.js';
+/**
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export declare const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export declare const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/**
+ * An implementation detail of the {@link ConcurrencySystem} — configure it through
+ * {@link ConcurrencySystemOptions} (`loadSignals`, `currentHistorySecs` and `snapshotHistorySecs`).
+ * @internal
+ */
+export interface SystemStatusOptions {
+    /**
+     * Defines max age of snapshots used in the {@link SystemStatus.getCurrentStatus} measurement.
+     * @default 5
+     */
+    currentHistorySecs?: number;
+    /**
+     * Defines max age of snapshots used in the {@link SystemStatus.getHistoricalStatus} measurement — the window
+     * autoscaling decisions are based on. Applied uniformly to every signal, built-in or custom, so that a signal's
+     * private retention cannot silently widen the window.
+     * @default 30
+     */
+    historySecs?: number;
+    /**
+     * The `Snapshotter` whose built-in signals are evaluated.
+     */
+    snapshotter: Snapshotter;
+    /**
+     * Additional load signals to include in the system status evaluation.
+     * These are evaluated alongside the built-in memory, CPU, event loop,
+     * and storage backend signals. If any signal reports overload, the system
+     * is considered overloaded. Each signal carries its own overload ratio.
+     */
+    loadSignals?: LoadSignal[];
+}
+export interface FinalStatistics {
+    requestsSucceeded: number;
+    requestsFailed: number;
+    retryHistogram: number[];
+    requestAvgFailedDurationMillis: number;
+    requestAvgSucceededDurationMillis: number;
+    requestsSucceededPerMinute: number;
+    requestsFailedPerMinute: number;
+    requestTotalDurationMillis: number;
+    requestsTotal: number;
+    crawlerRuntimeMillis: number;
+}
+/**
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
+ *
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
+ *
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
+ */
+export declare class SystemStatus {
+    #private;
+    constructor(options: SystemStatusOptions);
+    /**
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
+     */
+    get maxSampleWindowMillis(): number;
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    private assertUniqueSignalNames;
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the last `options.currentHistorySecs` seconds,
+     * and `true` otherwise.
+     */
+    getCurrentStatus(): SystemInfo;
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
+     */
+    getHistoricalStatus(): SystemInfo;
+    /**
+     * Returns a system status object.
+     */
+    private isSystemIdle;
+}

package/internals/autoscaling/system_status.js

@@ -0,0 +1,139 @@
+import { evaluateLoadSignalSample } from './load_signal.js';
+/**
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/** The four built-in signal names that map to typed `SystemInfo` fields, and the option that switches each off. */
+const BUILTIN_SIGNAL_OPTION_KEYS = {
+    memInfo: 'memory',
+    eventLoopInfo: 'eventLoop',
+    cpuInfo: 'cpu',
+    storageBackendInfo: 'storageBackend',
+};
+const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
+/**
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
+ *
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
+ *
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
+ */
+export class SystemStatus {
+    #currentHistoryMillis;
+    #historyMillis;
+    #signals;
+    constructor(options) {
+        const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
+        this.#currentHistoryMillis = currentHistorySecs * 1000;
+        this.#historyMillis = historySecs * 1000;
+        this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
+        this.assertUniqueSignalNames();
+    }
+    /**
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
+     */
+    get maxSampleWindowMillis() {
+        return Math.max(this.#currentHistoryMillis, this.#historyMillis);
+    }
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    assertUniqueSignalNames() {
+        const seen = new Set();
+        for (const { name } of this.#signals) {
+            if (!seen.has(name)) {
+                seen.add(name);
+                continue;
+            }
+            const hint = BUILTIN_SIGNAL_NAMES.has(name)
+                ? `it is the name of a built-in signal. To replace that signal, switch it off with \`loadSignals: { ${BUILTIN_SIGNAL_OPTION_KEYS[name]}: false }\` and keep your implementation in \`loadSignals.custom\`; to run yours alongside it, give it a different name.`
+                : 'two custom signals cannot share a name - rename one of them.';
+            throw new Error(`Duplicate load signal name ${JSON.stringify(name)}: ${hint}`);
+        }
+    }
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system
+     * has been overloaded in the last `options.currentHistorySecs` seconds,
+     * and `true` otherwise.
+     */
+    getCurrentStatus() {
+        return this.isSystemIdle(this.#currentHistoryMillis);
+    }
+    /**
+     * Returns an {@link SystemInfo} object with the following structure:
+     *
+     * ```javascript
+     * {
+     *     isSystemIdle: Boolean,
+     *     memInfo: Object,
+     *     eventLoopInfo: Object,
+     *     cpuInfo: Object
+     * }
+     * ```
+     *
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
+     */
+    getHistoricalStatus() {
+        return this.isSystemIdle(this.#historyMillis);
+    }
+    /**
+     * Returns a system status object.
+     */
+    isSystemIdle(sampleDurationMillis) {
+        const result = {
+            isSystemIdle: true,
+            memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+            storageBackendInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
+        };
+        let loadSignalInfo;
+        for (const signal of this.#signals) {
+            const sample = signal.getSample(sampleDurationMillis);
+            const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
+            if (info.isOverloaded) {
+                result.isSystemIdle = false;
+            }
+            if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
+                result[signal.name] = info;
+            }
+            else {
+                loadSignalInfo ??= {};
+                loadSignalInfo[signal.name] = info;
+            }
+        }
+        if (loadSignalInfo) {
+            result.loadSignalInfo = loadSignalInfo;
+        }
+        return result;
+    }
+}

package/internals/autoscaling/weighted_avg.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;

package/internals/autoscaling/weighted_avg.js

@@ -0,0 +1,14 @@
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export function weightedAvg(arrValues, arrWeights) {
+    const result = arrValues
+        .map((value, i) => {
+        const weight = arrWeights[i];
+        const sum = value * weight;
+        return [sum, weight];
+    })
+        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
+    return result[0] / result[1];
+}