@crawlee/core 4.0.0-beta.11 → 4.0.0-beta.111

package/autoscaling/load_signal.js

@@ -0,0 +1,104 @@
+import { weightedAvg } from './weighted_avg.js';
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
+ * and so can yours — it is the only part of their machinery worth reusing.
+ */
+export class SnapshotStore {
+    #snapshots = [];
+    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
+    // kept as TS-private: concurrency_system tests read this retention window directly
+    historyMillis = Infinity;
+    /**
+     * Sizes retention to the window the signal will be sampled over, as handed to it in
+     * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
+     * its start context grows unboundedly.
+     */
+    useSampleWindow(maxSampleWindowMillis) {
+        this.historyMillis = maxSampleWindowMillis;
+    }
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot, now = snapshot.createdAt) {
+        // Inline pruning to avoid private-method transpilation issues
+        let oldCount = 0;
+        for (let i = 0; i < this.#snapshots.length; i++) {
+            const { createdAt } = this.#snapshots[i];
+            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
+                oldCount++;
+            else
+                break;
+        }
+        if (oldCount)
+            this.#snapshots.splice(0, oldCount);
+        this.#snapshots.push(snapshot);
+    }
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis) {
+        if (!sampleDurationMillis)
+            return this.#snapshots;
+        const sample = [];
+        let idx = this.#snapshots.length;
+        if (!idx)
+            return sample;
+        const latestTime = this.#snapshots[idx - 1].createdAt;
+        while (idx--) {
+            const snapshot = this.#snapshots[idx];
+            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
+                sample.unshift(snapshot);
+            }
+            else {
+                break;
+            }
+        }
+        return sample;
+    }
+    /**
+     * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
+     * to compute a delta (e.g. the event loop and client signals read the last entry to measure change since it).
+     */
+    getAll() {
+        return this.#snapshots;
+    }
+    /**
+     * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
+     * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
+     * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
+     * rather than on stop leaves a finished session readable.
+     */
+    clear() {
+        this.#snapshots = [];
+    }
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ * @internal
+ */
+export function evaluateLoadSignalSample(sample, overloadedRatio) {
+    if (sample.length === 0) {
+        return {
+            isOverloaded: false,
+            limitRatio: overloadedRatio,
+            actualRatio: 0,
+        };
+    }
+    const weights = [];
+    const values = [];
+    for (let i = 1; i < sample.length; i++) {
+        const previous = sample[i - 1];
+        const current = sample[i];
+        const weight = +current.createdAt - +previous.createdAt;
+        weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
+        values.push(+current.isOverloaded);
+    }
+    const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
+    return {
+        isOverloaded: wAvg > overloadedRatio,
+        limitRatio: overloadedRatio,
+        actualRatio: Math.round(wAvg * 1000) / 1000,
+    };
+}

package/autoscaling/memory_load_signal.d.ts

@@ -0,0 +1,47 @@
+import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
+import type { SystemInfo } from './system_status.js';
+/**
+ * A snapshot produced by the built-in memory signal.
+ * @internal
+ */
+export interface MemorySnapshot extends LoadSnapshot {
+    usedBytes?: number;
+}
+/**
+ * Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
+ * {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
+ */
+export interface MemoryLoadSignalOptions {
+    /**
+     * Defines the maximum ratio of total memory that can be used.
+     * Exceeding this limit overloads the memory.
+     * @default 0.9
+     */
+    maxUsedRatio?: number;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
+     * @default 0.2
+     */
+    overloadedRatio?: number;
+}
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
+ * threshold. Also warns when memory use becomes critical.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export declare class MemoryLoadSignal implements LoadSignal {
+    #private;
+    readonly name = "memInfo";
+    readonly overloadedRatio: number;
+    constructor(options?: MemoryLoadSignalOptions);
+    start(context: LoadSignalStartContext): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
+    handle(systemInfo: SystemInfo): void;
+    private memoryOverloadWarning;
+    private getTotalMemoryBytes;
+}

package/autoscaling/memory_load_signal.js

@@ -0,0 +1,105 @@
+import { serviceLocator } from '../service_locator.js';
+import { getMemoryInfo } from '../system-info/memory-info.js';
+import { isContainerized } from '../system-info/runtime.js';
+import { SnapshotStore } from './load_signal.js';
+const RESERVE_MEMORY_RATIO = 0.5;
+const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
+/**
+ * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
+ * threshold. Also warns when memory use becomes critical.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * @category Scaling
+ */
+export class MemoryLoadSignal {
+    name = 'memInfo';
+    overloadedRatio;
+    #store = new SnapshotStore();
+    #maxUsedRatio;
+    /** All resolved in `start()`, before anything that reads them can fire. */
+    #config;
+    #log;
+    #maxMemoryBytes;
+    #events;
+    #maxMemoryRatio;
+    #lastLoggedCriticalMemoryOverloadAt = null;
+    constructor(options = {}) {
+        this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;
+        this.overloadedRatio = options.overloadedRatio ?? 0.2;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.#store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
+        this.#store.clear();
+        // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
+        // between systems) must not capture whichever services happened to be registered at that moment.
+        this.#config = serviceLocator.getConfiguration();
+        this.#events = serviceLocator.getEventManager();
+        this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
+        const memoryMbytes = this.#config.memoryMbytes ?? 0;
+        if (memoryMbytes > 0) {
+            this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;
+        }
+        else {
+            this.#maxMemoryRatio = this.#config.availableMemoryRatio;
+            if (!this.#maxMemoryRatio) {
+                throw new Error('availableMemoryRatio is not set in configuration.');
+            }
+            else {
+                this.#log.debug(`Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +
+                    'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
+            }
+            // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
+            this.#maxMemoryBytes = await this.getTotalMemoryBytes();
+        }
+        this.#events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+    }
+    async stop() {
+        this.#events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+        this.#events = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.#store.getSample(sampleDurationMillis);
+    }
+    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
+    handle(systemInfo) {
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        const { memCurrentBytes, memTotalBytes } = systemInfo;
+        let maxMemoryBytes = this.#maxMemoryBytes;
+        if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {
+            maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);
+        }
+        const snapshot = {
+            createdAt,
+            isOverloaded: memCurrentBytes / maxMemoryBytes > this.#maxUsedRatio,
+            usedBytes: memCurrentBytes,
+        };
+        this.#store.push(snapshot, createdAt);
+        this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
+    }
+    memoryOverloadWarning(systemInfo, maxMemoryBytes) {
+        const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
+        const { memCurrentBytes } = systemInfo;
+        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+        if (this.#lastLoggedCriticalMemoryOverloadAt &&
+            +createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+            return;
+        const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;
+        const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;
+        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
+        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
+        if (isCriticalOverload) {
+            const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
+            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
+            this.#log.warning('Memory is critically overloaded. ' +
+                `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
+            this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
+        }
+    }
+    async getTotalMemoryBytes() {
+        const containerized = this.#config.containerized ?? (await isContainerized());
+        return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
+    }
+}

package/autoscaling/snapshotter.d.ts

@@ -1,189 +1,84 @@
-import type { StorageClient } from '@crawlee/types';
-import type { Log } from '@apify/log';
-import type { BetterIntervalID } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
-import type { EventManager } from '../events/event_manager.js';
-import type { SystemInfo } from './system_status.js';
-export interface SnapshotterOptions {
-    /**
-     * Defines the interval of measuring the event loop response time.
-     * @default 0.5
-     */
-    eventLoopSnapshotIntervalSecs?: number;
+import type { ClientLoadSignalOptions } from './client_load_signal.js';
+import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
+import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
+import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
+import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
+/**
+ * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
+ *
+ * Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
+ * corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
+ * `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
+ * unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
+ *
+ * How far back the signals are evaluated is *not* set here, but by
+ * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
+ * {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
+ */
+export interface LoadSignalsOptions {
     /**
-     * Defines the interval of checking the current state
-     * of the remote API client.
-     * @default 1
+     * Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
+     * it off.
      */
-    clientSnapshotIntervalSecs?: number;
+    memory?: MemoryLoadSignalOptions | false;
     /**
-     * Maximum allowed delay of the event loop in milliseconds.
-     * Exceeding this limit overloads the event loop.
-     * @default 50
+     * Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
+     * ratio), or `false` to switch it off — which also stops its measuring interval.
      */
-    maxBlockedMillis?: number;
+    eventLoop?: EventLoopLoadSignalOptions | false;
     /**
-     * Defines the maximum ratio of total memory that can be used.
-     * Exceeding this limit overloads the memory.
-     * @default 0.9
+     * Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
      */
-    maxUsedMemoryRatio?: number;
+    cpu?: CpuLoadSignalOptions | false;
     /**
-     * Defines the maximum number of new rate limit errors within
-     * the given interval.
-     * @default 1
+     * Tuning for the built-in {@link ClientLoadSignal} (snapshot interval + error limit + overload ratio), or
+     * `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
+     * signal otherwise polls it every second to no purpose.
      */
-    maxClientErrors?: number;
+    client?: ClientLoadSignalOptions | false;
     /**
-     * Sets the interval in seconds for which a history of resource snapshots
-     * will be kept. Increasing this to very high numbers will affect performance.
-     * @default 60
+     * Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
+     * alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
+     * is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
+     * not collide with an enabled built-in's.
      */
-    snapshotHistorySecs?: number;
-    /** @internal */
-    log?: Log;
-    /** @internal */
-    client?: StorageClient;
-    /** @internal */
-    config?: Configuration;
-}
-interface MemorySnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedBytes?: number;
-}
-interface CpuSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    usedRatio: number;
-    ticks?: {
-        idle: number;
-        total: number;
-    };
-}
-interface EventLoopSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    exceededMillis: number;
-}
-interface ClientSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-    rateLimitErrorCount: number;
+    custom?: LoadSignal[];
 }
 /**
- * Creates snapshots of system resources at given intervals and marks the resource
- * as either overloaded or not during the last interval. Keeps a history of the snapshots.
- * It tracks the following resources: Memory, EventLoop, API and CPU.
- * The class is used by the {@link AutoscaledPool} class.
- *
- * When running on the Apify platform, the CPU and memory statistics are provided by the platform,
- * as collected from the running Docker container. When running locally, `Snapshotter`
- * makes its own statistics by querying the OS.
- *
- * CPU becomes overloaded locally when its current use exceeds the `maxUsedCpuRatio` option or
- * when Apify platform marks it as overloaded.
- *
- * Memory becomes overloaded if its current use exceeds the `maxUsedMemoryRatio` option.
- * It's computed using the total memory available to the container when running on
- * the Apify platform and a quarter of total system memory when running locally.
- * Max total memory when running locally may be overridden by using the `CRAWLEE_MEMORY_MBYTES`
- * environment variable.
- *
- * Event loop becomes overloaded if it slows down by more than the `maxBlockedMillis` option.
+ * An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
+ * {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
+ * them here.
+ * @internal
+ */
+export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
+/**
+ * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
+ * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link ClientLoadSignal} — constructing the ones
+ * that were not switched off and driving their shared lifecycle.
  *
- * Client becomes overloaded when rate limit errors (429 - Too Many Requests),
- * typically received from the request queue, exceed the set limit within the set interval.
- * @category Scaling
+ * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
+ * are simply forwarded to the corresponding constructor.
+ * @internal
  */
 export declare class Snapshotter {
-    log: Log;
-    client: StorageClient;
-    config: Configuration;
-    events: EventManager;
-    eventLoopSnapshotIntervalMillis: number;
-    clientSnapshotIntervalMillis: number;
-    snapshotHistoryMillis: number;
-    maxBlockedMillis: number;
-    maxUsedMemoryRatio: number;
-    maxClientErrors: number;
-    maxMemoryBytes: number;
-    cpuSnapshots: CpuSnapshot[];
-    eventLoopSnapshots: EventLoopSnapshot[];
-    memorySnapshots: MemorySnapshot[];
-    clientSnapshots: ClientSnapshot[];
-    eventLoopInterval: BetterIntervalID;
-    clientInterval: BetterIntervalID;
-    lastLoggedCriticalMemoryOverloadAt: Date | null;
+    #private;
+    /**
+     * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
+     * instances. Signals switched off through the options are simply absent — the system status reports them as
+     * not overloaded.
+     */
+    getLoadSignals(): LoadSignal[];
     /**
      * @param [options] All `Snapshotter` configuration options.
      */
     constructor(options?: SnapshotterOptions);
     /**
-     * Starts capturing snapshots at configured intervals.
+     * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
+     * be queried with, which is also how much history they retain.
      */
-    start(): Promise<void>;
+    start(context: LoadSignalStartContext): Promise<void>;
     /**
      * Stops all resource capturing.
      */
     stop(): Promise<void>;
-    /**
-     * Returns a sample of latest memory snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getMemorySample(sampleDurationMillis?: number): MemorySnapshot[];
-    /**
-     * Returns a sample of latest event loop snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getEventLoopSample(sampleDurationMillis?: number): EventLoopSnapshot[];
-    /**
-     * Returns a sample of latest CPU snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getCpuSample(sampleDurationMillis?: number): CpuSnapshot[];
-    /**
-     * Returns a sample of latest Client snapshots, with the size of the sample defined
-     * by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
-     */
-    getClientSample(sampleDurationMillis?: number): ClientSnapshot[];
-    /**
-     * Finds the latest snapshots by sampleDurationMillis in the provided array.
-     */
-    protected _getSample<T extends {
-        createdAt: Date;
-    }>(snapshots: T[], sampleDurationMillis?: number): T[];
-    /**
-     * Creates a snapshot of current memory usage
-     * using the Apify platform `systemInfo` event.
-     */
-    protected _snapshotMemory(systemInfo: SystemInfo): void;
-    /**
-     * Checks for critical memory overload and logs it to the console.
-     */
-    protected _memoryOverloadWarning(systemInfo: SystemInfo): void;
-    /**
-     * Creates a snapshot of current event loop delay.
-     */
-    protected _snapshotEventLoop(intervalCallback: () => unknown): void;
-    /**
-     * Creates a snapshot of current CPU usage using the Apify platform `systemInfo` event.
-     */
-    protected _snapshotCpu(systemInfo: SystemInfo): void;
-    /**
-     * Creates a snapshot of current API state by checking for
-     * rate limit errors. Only errors produced by a 2nd retry
-     * of the API call are considered for snapshotting since
-     * earlier errors may just be caused by a random spike in
-     * number of requests and do not necessarily signify API
-     * overloading.
-     */
-    protected _snapshotClient(intervalCallback: () => unknown): void;
-    /**
-     * Removes snapshots that are older than the snapshotHistorySecs option
-     * from the array (destructively - in place).
-     */
-    protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date): void;
 }
-export {};
-//# sourceMappingURL=snapshotter.d.ts.map