@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/autoscaling/autoscaled_pool.d.ts

@@ -1,5 +1,4 @@
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import type { SnapshotterOptions } from './snapshotter.js';
 import type { SystemInfo, SystemStatusOptions } from './system_status.js';
 export interface AutoscaledPoolOptions {
@@ -99,7 +98,7 @@ export interface AutoscaledPoolOptions {
      * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
      */
     maxTasksPerMinute?: number;
-    log?: Log;
+    log?: CrawleeLogger;
 }
 /**
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
@@ -150,7 +149,6 @@ export interface AutoscaledPoolOptions {
  * @category Scaling
  */
 export declare class AutoscaledPool {
-    private readonly config;
     private readonly log;
     private readonly desiredConcurrencyRatio;
     private readonly scaleUpStepRatio;
@@ -172,6 +170,8 @@ export declare class AutoscaledPool {
     private resolve;
     private reject;
     private snapshotter;
+    /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
+    private loadSignals;
     private systemStatus;
     private autoscaleInterval;
     private maybeRunInterval;
@@ -179,7 +179,7 @@ export declare class AutoscaledPool {
     private queryingIsFinished;
     private tasksDonePerSecondInterval?;
     private _tasksPerMinute;
-    constructor(options: AutoscaledPoolOptions, config?: Configuration);
+    constructor(options: AutoscaledPoolOptions);
     /**
      * Gets the minimum number of tasks running in parallel.
      */
@@ -297,4 +297,3 @@ export declare class AutoscaledPool {
     protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
     protected get _isOverMaxRequestLimit(): boolean;
 }
-//# sourceMappingURL=autoscaled_pool.d.ts.map

package/autoscaling/autoscaled_pool.js

@@ -1,9 +1,8 @@
 import ow from 'ow';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
 import { CriticalError } from '../errors.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { Snapshotter } from './snapshotter.js';
 import { SystemStatus } from './system_status.js';
 /**
@@ -55,7 +54,6 @@ import { SystemStatus } from './system_status.js';
  * @category Scaling
  */
 export class AutoscaledPool {
-    config;
     log;
     // Configurable properties.
     desiredConcurrencyRatio;
@@ -79,6 +77,8 @@ export class AutoscaledPool {
     resolve = null;
     reject = null;
     snapshotter;
+    /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
+    loadSignals;
     systemStatus;
     autoscaleInterval;
     maybeRunInterval;
@@ -86,8 +86,7 @@ export class AutoscaledPool {
     queryingIsFinished;
     tasksDonePerSecondInterval;
     _tasksPerMinute = Array.from({ length: 60 }, () => 0);
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        this.config = config;
+    constructor(options) {
         ow(options, ow.object.exactShape({
             runTaskFunction: ow.function,
             isFinishedFunction: ow.function,
@@ -107,7 +106,7 @@ export class AutoscaledPool {
             log: ow.optional.object,
             maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
         }));
-        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = defaultLog, maxTasksPerMinute = Infinity, } = options;
+        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = serviceLocator.getLogger(), maxTasksPerMinute = Infinity, } = options;
         this.log = log.child({ prefix: 'AutoscaledPool' });
         // Configurable properties.
         this.desiredConcurrencyRatio = desiredConcurrencyRatio;
@@ -137,11 +136,9 @@ export class AutoscaledPool {
         ssoCopy.snapshotter ??= new Snapshotter({
             ...snapshotterOptions,
             log: this.log,
-            config: this.config,
-            client: this.config.getStorageClient(),
         });
-        ssoCopy.config ??= this.config;
         this.snapshotter = ssoCopy.snapshotter;
+        this.loadSignals = ssoCopy.loadSignals ?? [];
         this.systemStatus = new SystemStatus(ssoCopy);
     }
     /**
@@ -204,6 +201,7 @@ export class AutoscaledPool {
             this.reject = reject;
         });
         await this.snapshotter.start();
+        await Promise.all(this.loadSignals.map((s) => s.start()));
         // This interval checks the system status and updates the desired concurrency accordingly.
         this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
         // This is here because if we scale down to let's say 1, then after each promise is finished
@@ -258,14 +256,17 @@ export class AutoscaledPool {
         this.isStopped = true;
         await new Promise((resolve, reject) => {
             let timeout;
+            let interval;
             if (timeoutSecs) {
                 timeout = setTimeout(() => {
+                    // Clean up the polling interval to prevent it from leaking on timeout.
+                    clearInterval(interval);
                     const err = new Error("The pool's running tasks did not finish" +
                         `in ${timeoutSecs} secs after pool.pause() invocation.`);
                     reject(err);
                 }, timeoutSecs);
             }
-            const interval = setInterval(() => {
+            interval = setInterval(() => {
                 if (this._currentConcurrency <= 0) {
                     // Clean up timeout and interval to prevent process hanging.
                     if (timeout)
@@ -517,6 +518,7 @@ export class AutoscaledPool {
             betterClearInterval(this.tasksDonePerSecondInterval);
         if (this.snapshotter)
             await this.snapshotter.stop();
+        await Promise.all(this.loadSignals.map((s) => s.stop()));
     }
     _incrementTasksDonePerSecond(intervalCallback) {
         this._tasksPerMinute.unshift(0);
@@ -530,4 +532,3 @@ export class AutoscaledPool {
         return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
     }
 }
-//# sourceMappingURL=autoscaled_pool.js.map

package/autoscaling/client_load_signal.d.ts

@@ -0,0 +1,25 @@
+import type { StorageBackend } from '@crawlee/types';
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+export interface ClientSnapshot extends LoadSnapshot {
+    rateLimitErrorCount: number;
+}
+export interface ClientLoadSignalOptions {
+    client: StorageBackend;
+    clientSnapshotIntervalSecs?: number;
+    maxClientErrors?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+}
+/**
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
+ * and reports overload when the error delta exceeds a threshold.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createClientLoadSignal(options: ClientLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<ClientSnapshot>;
+    handle: (cb: () => unknown) => void;
+    getSample(sampleDurationMillis?: number): ClientSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type ClientLoadSignal = ReturnType<typeof createClientLoadSignal>;

package/autoscaling/client_load_signal.js

@@ -0,0 +1,36 @@
+import { SnapshotStore } from './load_signal.js';
+const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+/**
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
+ * and reports overload when the error delta exceeds a threshold.
+ */
+export function createClientLoadSignal(options) {
+    const maxClientErrors = options.maxClientErrors ?? 3;
+    const signal = SnapshotStore.fromInterval({
+        name: 'clientInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.3,
+        intervalMillis: (options.clientSnapshotIntervalSecs ?? 1) * 1000,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, intervalCallback) {
+            const now = new Date();
+            const allErrorCounts = options.client.stats?.rateLimitErrors ?? [];
+            const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+            const snapshot = {
+                createdAt: now,
+                isOverloaded: false,
+                rateLimitErrorCount: currentErrCount,
+            };
+            const all = store.getAll();
+            const previousSnapshot = all[all.length - 1];
+            if (previousSnapshot) {
+                const { rateLimitErrorCount } = previousSnapshot;
+                const delta = currentErrCount - rateLimitErrorCount;
+                if (delta > maxClientErrors)
+                    snapshot.isOverloaded = true;
+            }
+            store.push(snapshot, now);
+            intervalCallback();
+        },
+    });
+    return signal;
+}

package/autoscaling/cpu_load_signal.d.ts

@@ -0,0 +1,28 @@
+import type { Configuration } from '../configuration.js';
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+import type { SystemInfo } from './system_status.js';
+export interface CpuSnapshot extends LoadSnapshot {
+    usedRatio: number;
+    ticks?: {
+        idle: number;
+        total: number;
+    };
+}
+export interface CpuLoadSignalOptions {
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+    config: Configuration;
+}
+/**
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
+ * the platform or local OS metrics indicate the CPU is overloaded.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<CpuSnapshot>;
+    handle: (payload: SystemInfo) => void;
+    getSample(sampleDurationMillis?: number): CpuSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;

package/autoscaling/cpu_load_signal.js

@@ -0,0 +1,24 @@
+import { serviceLocator } from '../service_locator.js';
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
+ * the platform or local OS metrics indicate the CPU is overloaded.
+ */
+export function createCpuLoadSignal(options) {
+    return SnapshotStore.fromEvent({
+        name: 'cpuInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.4,
+        events: serviceLocator.getEventManager(),
+        event: "systemInfo" /* EventType.SYSTEM_INFO */,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, systemInfo) {
+            const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
+            const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+            store.push({
+                createdAt,
+                isOverloaded: isCpuOverloaded,
+                usedRatio: Math.ceil(cpuCurrentUsage / 100),
+            }, createdAt);
+        },
+    });
+}

package/autoscaling/event_loop_load_signal.d.ts

@@ -0,0 +1,23 @@
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+export interface EventLoopSnapshot extends LoadSnapshot {
+    exceededMillis: number;
+}
+export interface EventLoopLoadSignalOptions {
+    eventLoopSnapshotIntervalSecs?: number;
+    maxBlockedMillis?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+}
+/**
+ * Periodically measures event loop delay and reports overload when the
+ * delay exceeds a configured threshold.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<EventLoopSnapshot>;
+    handle: (cb: () => unknown) => void;
+    getSample(sampleDurationMillis?: number): EventLoopSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;

package/autoscaling/event_loop_load_signal.js

@@ -0,0 +1,35 @@
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Periodically measures event loop delay and reports overload when the
+ * delay exceeds a configured threshold.
+ */
+export function createEventLoopLoadSignal(options = {}) {
+    const intervalMillis = (options.eventLoopSnapshotIntervalSecs ?? 0.5) * 1000;
+    const maxBlockedMillis = options.maxBlockedMillis ?? 50;
+    const signal = SnapshotStore.fromInterval({
+        name: 'eventLoopInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.6,
+        intervalMillis,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, intervalCallback) {
+            const now = new Date();
+            const snapshot = {
+                createdAt: now,
+                isOverloaded: false,
+                exceededMillis: 0,
+            };
+            const all = store.getAll();
+            const previousSnapshot = all[all.length - 1];
+            if (previousSnapshot) {
+                const { createdAt } = previousSnapshot;
+                const delta = now.getTime() - +createdAt - intervalMillis;
+                if (delta > maxBlockedMillis)
+                    snapshot.isOverloaded = true;
+                snapshot.exceededMillis = Math.max(delta - maxBlockedMillis, 0);
+            }
+            store.push(snapshot, now);
+            intervalCallback();
+        },
+    });
+    return signal;
+}

package/autoscaling/index.d.ts

@@ -1,4 +1,8 @@
 export * from './autoscaled_pool.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.d.ts.map

package/autoscaling/index.js

@@ -1,4 +1,8 @@
 export * from './autoscaled_pool.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.js.map

package/autoscaling/load_signal.d.ts

@@ -0,0 +1,99 @@
+import type { EventManager, EventTypeName } from '../events/event_manager.js';
+import type { ClientInfo } from './system_status.js';
+/**
+ * A snapshot of a resource's overload state at a point in time.
+ */
+export interface LoadSnapshot {
+    createdAt: Date;
+    isOverloaded: boolean;
+}
+/**
+ * A signal that reports whether a particular resource is overloaded.
+ *
+ * `SystemStatus` aggregates multiple `LoadSignal` instances to determine
+ * overall system health. The built-in signals cover memory, CPU, event loop,
+ * and API client rate limits. You can implement this interface to add
+ * custom overload signals (e.g. navigation timeouts, proxy health).
+ */
+export interface LoadSignal {
+    /** Human-readable name used in logging and `SystemInfo` keys. */
+    readonly name: string;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before the signal
+     * is considered overloaded. For example, `0.2` means the signal fires
+     * when more than 20% of the sample window is overloaded.
+     */
+    readonly overloadedRatio: number;
+    /** Start collecting snapshots. Called when the pool starts. */
+    start(): Promise<void>;
+    /** Stop collecting snapshots. Called when the pool shuts down. */
+    stop(): Promise<void>;
+    /**
+     * Return snapshots for a recent time window (used for "current" status).
+     * @param sampleDurationMillis How far back to look, in milliseconds.
+     */
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+}
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
+ * Signals compose with this instead of inheriting from a base class.
+ */
+export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
+    private snapshots;
+    private readonly historyMillis;
+    constructor(historyMillis?: number);
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot: T, now?: Date): void;
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis?: number): T[];
+    /**
+     * Direct access to the underlying array (for backward-compat getters).
+     */
+    getAll(): T[];
+    /**
+     * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
+     *
+     * The `handler` receives the store (to read previous snapshots) and the
+     * interval callback (which it **must** call when done). It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromInterval<T extends LoadSnapshot>(options: {
+        name: string;
+        overloadedRatio: number;
+        intervalMillis: number;
+        snapshotHistoryMillis?: number;
+        handler: (store: SnapshotStore<T>, intervalCallback: () => unknown) => void;
+    }): Omit<LoadSignal, 'getSample'> & {
+        store: SnapshotStore<T>;
+        handle: (cb: () => unknown) => void;
+        getSample(sampleDurationMillis?: number): T[];
+    };
+    /**
+     * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
+     *
+     * The `handler` receives the event payload and the store. It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromEvent<T extends LoadSnapshot, E>(options: {
+        name: string;
+        overloadedRatio: number;
+        events: EventManager;
+        event: EventTypeName;
+        snapshotHistoryMillis?: number;
+        handler: (store: SnapshotStore<T>, payload: E) => void;
+    }): Omit<LoadSignal, 'getSample'> & {
+        store: SnapshotStore<T>;
+        handle: (payload: E) => void;
+        getSample(sampleDurationMillis?: number): T[];
+    };
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ */
+export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): ClientInfo;

package/autoscaling/load_signal.js

@@ -0,0 +1,135 @@
+import { weightedAvg } from '@crawlee/utils';
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+/**
+ * A time-pruning, time-windowed store for `LoadSnapshot` values.
+ * Signals compose with this instead of inheriting from a base class.
+ */
+export class SnapshotStore {
+    snapshots = [];
+    historyMillis;
+    constructor(historyMillis = 30_000) {
+        this.historyMillis = historyMillis;
+    }
+    /**
+     * Add a snapshot and prune entries older than the history window.
+     */
+    push(snapshot, now = snapshot.createdAt) {
+        // Inline pruning to avoid private-method transpilation issues
+        let oldCount = 0;
+        for (let i = 0; i < this.snapshots.length; i++) {
+            const { createdAt } = this.snapshots[i];
+            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
+                oldCount++;
+            else
+                break;
+        }
+        if (oldCount)
+            this.snapshots.splice(0, oldCount);
+        this.snapshots.push(snapshot);
+    }
+    /**
+     * Return all snapshots, or only those within the given time window.
+     */
+    getSample(sampleDurationMillis) {
+        if (!sampleDurationMillis)
+            return this.snapshots;
+        const sample = [];
+        let idx = this.snapshots.length;
+        if (!idx)
+            return sample;
+        const latestTime = this.snapshots[idx - 1].createdAt;
+        while (idx--) {
+            const snapshot = this.snapshots[idx];
+            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
+                sample.unshift(snapshot);
+            }
+            else {
+                break;
+            }
+        }
+        return sample;
+    }
+    /**
+     * Direct access to the underlying array (for backward-compat getters).
+     */
+    getAll() {
+        return this.snapshots;
+    }
+    /**
+     * Create a `LoadSignal` that snapshots on a `betterSetInterval` tick.
+     *
+     * The `handler` receives the store (to read previous snapshots) and the
+     * interval callback (which it **must** call when done). It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromInterval(options) {
+        const store = new SnapshotStore(options.snapshotHistoryMillis);
+        let interval = null;
+        const handle = (cb) => options.handler(store, cb);
+        return {
+            name: options.name,
+            overloadedRatio: options.overloadedRatio,
+            store,
+            handle,
+            getSample: (ms) => store.getSample(ms),
+            async start() {
+                interval = betterSetInterval(handle, options.intervalMillis);
+            },
+            async stop() {
+                betterClearInterval(interval);
+            },
+        };
+    }
+    /**
+     * Create a `LoadSignal` that snapshots in response to an `EventManager` event.
+     *
+     * The `handler` receives the event payload and the store. It should call
+     * `store.push()` to record a snapshot.
+     */
+    static fromEvent(options) {
+        const store = new SnapshotStore(options.snapshotHistoryMillis);
+        const handle = (payload) => options.handler(store, payload);
+        return {
+            name: options.name,
+            overloadedRatio: options.overloadedRatio,
+            store,
+            handle,
+            getSample: (ms) => store.getSample(ms),
+            async start() {
+                options.events.on(options.event, handle);
+            },
+            async stop() {
+                options.events.off(options.event, handle);
+            },
+        };
+    }
+}
+/**
+ * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
+ * overloaded ratio, using a time-weighted average. This is the shared
+ * evaluation logic used by `SystemStatus` for all signal types.
+ */
+export function evaluateLoadSignalSample(sample, overloadedRatio) {
+    if (sample.length === 0) {
+        return {
+            isOverloaded: false,
+            limitRatio: overloadedRatio,
+            actualRatio: 0,
+        };
+    }
+    const weights = [];
+    const values = [];
+    for (let i = 1; i < sample.length; i++) {
+        const previous = sample[i - 1];
+        const current = sample[i];
+        const weight = +current.createdAt - +previous.createdAt;
+        weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
+        values.push(+current.isOverloaded);
+    }
+    const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
+    return {
+        isOverloaded: wAvg > overloadedRatio,
+        limitRatio: overloadedRatio,
+        actualRatio: Math.round(wAvg * 1000) / 1000,
+    };
+}