@crawlee/core 4.0.0-beta.87 → 4.0.0-beta.89

package/autoscaling/system_status.js

@@ -1,62 +1,72 @@
-import ow from 'ow';
 import { evaluateLoadSignalSample } from './load_signal.js';
-import { Snapshotter } from './snapshotter.js';
-/** The four built-in signal names that map to typed `SystemInfo` fields. */
-const BUILTIN_SIGNAL_NAMES = new Set(['memInfo', 'eventLoopInfo', 'cpuInfo', 'clientInfo']);
 /**
- * Provides a simple interface to reading system status from a {@link Snapshotter} instance.
- * It only exposes two functions {@link SystemStatus.getCurrentStatus}
- * and {@link SystemStatus.getHistoricalStatus}.
- * The system status is calculated using a weighted average of overloaded
- * messages in the snapshots, with the weights being the time intervals
- * between the snapshots. Each resource is calculated separately
- * and the system is overloaded whenever at least one resource is overloaded.
- * The class is used by the {@link AutoscaledPool} class.
+ * How far back the *current* system status looks by default — the window that gates task dispatch.
+ * @internal
+ */
+export const DEFAULT_CURRENT_HISTORY_SECS = 5;
+/**
+ * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
+ * therefore how much history the signals are asked to retain.
+ * @internal
+ */
+export const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
+/** The four built-in signal names that map to typed `SystemInfo` fields, and the option that switches each off. */
+const BUILTIN_SIGNAL_OPTION_KEYS = {
+    memInfo: 'memory',
+    eventLoopInfo: 'eventLoop',
+    cpuInfo: 'cpu',
+    clientInfo: 'client',
+};
+const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
+/**
+ * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
+ * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
+ * is overloaded whenever at least one of them is.
  *
- * {@link SystemStatus.getCurrentStatus}
- * returns a boolean that represents the current status of the system.
- * The length of the current timeframe in seconds is configurable
- * by the `currentHistorySecs` option and represents the max age
- * of snapshots to be considered for the calculation.
+ * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
+ * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
+ * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
  *
- * {@link SystemStatus.getHistoricalStatus}
- * returns a boolean that represents the long-term status
- * of the system. It considers the full snapshot history available
- * in the {@link Snapshotter} instance.
- * @category Scaling
+ * An implementation detail of the {@link ConcurrencySystem}, configured through
+ * {@link ConcurrencySystemOptions}.
+ * @internal
  */
 export class SystemStatus {
     currentHistoryMillis;
-    snapshotter;
+    historyMillis;
     signals;
+    constructor(options) {
+        const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
+        this.currentHistoryMillis = currentHistorySecs * 1000;
+        this.historyMillis = historySecs * 1000;
+        this.signals = [...snapshotter.getLoadSignals(), ...loadSignals];
+        this.assertUniqueSignalNames();
+    }
     /**
-     * Per-signal ratio overrides. The built-in four get their overrides from
-     * the legacy `max*OverloadedRatio` options; custom signals use their own
-     * `overloadedRatio`.
+     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
+     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
+     * defaults.
      */
-    ratioOverrides;
-    constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            currentHistorySecs: ow.optional.number,
-            maxMemoryOverloadedRatio: ow.optional.number,
-            maxEventLoopOverloadedRatio: ow.optional.number,
-            maxCpuOverloadedRatio: ow.optional.number,
-            maxClientOverloadedRatio: ow.optional.number,
-            snapshotter: ow.optional.object,
-            loadSignals: ow.optional.array,
-        }));
-        const { currentHistorySecs = 5, maxMemoryOverloadedRatio = 0.2, maxEventLoopOverloadedRatio = 0.6, maxCpuOverloadedRatio = 0.4, maxClientOverloadedRatio = 0.3, snapshotter, loadSignals = [], } = options;
-        this.currentHistoryMillis = currentHistorySecs * 1000;
-        this.snapshotter = snapshotter || new Snapshotter();
-        // Built-in signals from the snapshotter + any custom signals
-        this.signals = [...this.snapshotter.getLoadSignals(), ...loadSignals];
-        // Allow legacy options to override the built-in signal ratios
-        this.ratioOverrides = {
-            memInfo: maxMemoryOverloadedRatio,
-            eventLoopInfo: maxEventLoopOverloadedRatio,
-            cpuInfo: maxCpuOverloadedRatio,
-            clientInfo: maxClientOverloadedRatio,
-        };
+    get maxSampleWindowMillis() {
+        return Math.max(this.currentHistoryMillis, this.historyMillis);
+    }
+    /**
+     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
+     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
+     * only the last is reported.
+     */
+    assertUniqueSignalNames() {
+        const seen = new Set();
+        for (const { name } of this.signals) {
+            if (!seen.has(name)) {
+                seen.add(name);
+                continue;
+            }
+            const hint = BUILTIN_SIGNAL_NAMES.has(name)
+                ? `it is the name of a built-in signal. To replace that signal, switch it off with \`loadSignals: { ${BUILTIN_SIGNAL_OPTION_KEYS[name]}: false }\` and keep your implementation in \`loadSignals.custom\`; to run yours alongside it, give it a different name.`
+                : 'two custom signals cannot share a name - rename one of them.';
+            throw new Error(`Duplicate load signal name ${JSON.stringify(name)}: ${hint}`);
+        }
     }
     /**
      * Returns an {@link SystemInfo} object with the following structure:
@@ -89,12 +99,11 @@ export class SystemStatus {
      * }
      * ```
      *
-     * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the full history of the {@link Snapshotter}
-     * (which is configurable in the {@link Snapshotter}) and `true` otherwise.
+     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
+     * `historySecs` seconds and `true` otherwise.
      */
     getHistoricalStatus() {
-        return this.isSystemIdle();
+        return this.isSystemIdle(this.historyMillis);
     }
     /**
      * Returns a system status object.
@@ -109,9 +118,8 @@ export class SystemStatus {
         };
         let loadSignalInfo;
         for (const signal of this.signals) {
-            const ratio = this.ratioOverrides[signal.name] ?? signal.overloadedRatio;
             const sample = signal.getSample(sampleDurationMillis);
-            const info = evaluateLoadSignalSample(sample, ratio);
+            const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
             if (info.isOverloaded) {
                 result.isSystemIdle = false;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.87",
+    "version": "4.0.0-beta.89",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.87",
-        "@crawlee/types": "4.0.0-beta.87",
-        "@crawlee/utils": "4.0.0-beta.87",
+        "@crawlee/fs-storage": "4.0.0-beta.89",
+        "@crawlee/types": "4.0.0-beta.89",
+        "@crawlee/utils": "4.0.0-beta.89",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "1b7604cd77b694460aac4448b4555444fa1b2bdb"
+    "gitHead": "9279941162eb4be0a9113f768b5bd79d27e66eba"
 }