@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/autoscaling/load_signal.d.ts

@@ -1,100 +0,0 @@
-import type { LoadSignalInfo } from './system_status.js';
-/**
- * A snapshot of a resource's overload state at a point in time.
- */
-export interface LoadSnapshot {
-    createdAt: Date;
-    isOverloaded: boolean;
-}
-/**
- * Handed to a {@link LoadSignal} when it starts, so it can size its snapshot retention to what it will actually
- * be asked for — without having to know how the {@link ConcurrencySystem} that drives it is configured.
- */
-export interface LoadSignalStartContext {
-    /**
-     * The longest sample window the signal will be queried with (the wider of the task-gating and autoscaling
-     * windows). Keeping less history than this contributes a narrower view of the resource than the other signals;
-     * keeping more is wasted memory, as the extra snapshots are never sampled.
-     */
-    maxSampleWindowMillis: number;
-}
-/**
- * A signal that reports whether a particular resource is overloaded. The {@link ConcurrencySystem} aggregates
- * several of them — if any one reports overload, the system is overloaded.
- *
- * The built-in signals cover memory, CPU, event loop and storage backend rate limits. Implement this interface to add
- * your own (navigation timeouts, proxy health, …) and pass them via
- * {@link LoadSignalsOptions.custom|`loadSignals.custom`}; {@link SnapshotStore} does the time-windowed
- * bookkeeping if you want it. Each built-in is also a public class, so one can be *wrapped* rather than reimplemented
- * — construct it yourself and switch the default off with {@link LoadSignalsOptions.cpu|`cpu: false`} or friends.
- */
-export interface LoadSignal {
-    /**
-     * This signal's key in the reported {@link SystemInfo}, also used in logging — so it must be unique among the
-     * signals of one {@link ConcurrencySystem}, which throws on a duplicate. The four built-in names (`memInfo`,
-     * `eventLoopInfo`, `cpuInfo`, `storageBackendInfo`) land in the correspondingly named `SystemInfo` fields rather
-     * than the `loadSignalInfo` bag; taking one over means switching that built-in off.
-     */
-    readonly name: string;
-    /**
-     * Maximum ratio of overloaded snapshots in a sample before the signal
-     * is considered overloaded. For example, `0.2` means the signal fires
-     * when more than 20% of the sample window is overloaded.
-     */
-    readonly overloadedRatio: number;
-    /**
-     * Start collecting snapshots, retaining at least the sample window named in the `context`. Called when the
-     * {@link ConcurrencySystem} starts — which may be a *restart*, so drop anything measured before it.
-     */
-    start(context: LoadSignalStartContext): Promise<void>;
-    /** Stop collecting snapshots. Called when the {@link ConcurrencySystem} shuts down. */
-    stop(): Promise<void>;
-    /**
-     * Return snapshots for a recent time window (used for "current" status).
-     * @param sampleDurationMillis How far back to look, in milliseconds.
-     */
-    getSample(sampleDurationMillis?: number): LoadSnapshot[];
-}
-/**
- * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
- * and so can yours — it is the only part of their machinery worth reusing.
- */
-export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
-    #private;
-    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
-    private historyMillis;
-    /**
-     * Sizes retention to the window the signal will be sampled over, as handed to it in
-     * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
-     * its start context grows unboundedly.
-     */
-    useSampleWindow(maxSampleWindowMillis: number): void;
-    /**
-     * Add a snapshot and prune entries older than the history window.
-     */
-    push(snapshot: T, now?: Date): void;
-    /**
-     * Return all snapshots, or only those within the given time window.
-     */
-    getSample(sampleDurationMillis?: number): T[];
-    /**
-     * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
-     * to compute a delta (e.g. the event loop and storage backend signals read the last entry to measure change since
-     * it).
-     */
-    getAll(): T[];
-    /**
-     * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
-     * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
-     * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
-     * rather than on stop leaves a finished session readable.
-     */
-    clear(): void;
-}
-/**
- * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
- * overloaded ratio, using a time-weighted average. This is the shared
- * evaluation logic used by `SystemStatus` for all signal types.
- * @internal
- */
-export declare function evaluateLoadSignalSample(sample: LoadSnapshot[], overloadedRatio: number): LoadSignalInfo;

package/autoscaling/load_signal.js

@@ -1,105 +0,0 @@
-import { weightedAvg } from './weighted_avg.js';
-/**
- * A time-pruning, time-windowed store for `LoadSnapshot` values. All four built-in signals compose with one of these,
- * and so can yours — it is the only part of their machinery worth reusing.
- */
-export class SnapshotStore {
-    #snapshots = [];
-    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
-    // kept as TS-private: concurrency_system tests read this retention window directly
-    historyMillis = Infinity;
-    /**
-     * Sizes retention to the window the signal will be sampled over, as handed to it in
-     * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
-     * its start context grows unboundedly.
-     */
-    useSampleWindow(maxSampleWindowMillis) {
-        this.historyMillis = maxSampleWindowMillis;
-    }
-    /**
-     * Add a snapshot and prune entries older than the history window.
-     */
-    push(snapshot, now = snapshot.createdAt) {
-        // Inline pruning to avoid private-method transpilation issues
-        let oldCount = 0;
-        for (let i = 0; i < this.#snapshots.length; i++) {
-            const { createdAt } = this.#snapshots[i];
-            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
-                oldCount++;
-            else
-                break;
-        }
-        if (oldCount)
-            this.#snapshots.splice(0, oldCount);
-        this.#snapshots.push(snapshot);
-    }
-    /**
-     * Return all snapshots, or only those within the given time window.
-     */
-    getSample(sampleDurationMillis) {
-        if (!sampleDurationMillis)
-            return this.#snapshots;
-        const sample = [];
-        let idx = this.#snapshots.length;
-        if (!idx)
-            return sample;
-        const latestTime = this.#snapshots[idx - 1].createdAt;
-        while (idx--) {
-            const snapshot = this.#snapshots[idx];
-            if (+latestTime - +snapshot.createdAt <= sampleDurationMillis) {
-                sample.unshift(snapshot);
-            }
-            else {
-                break;
-            }
-        }
-        return sample;
-    }
-    /**
-     * Direct, unwindowed access to the underlying array — used by signals whose handler needs the previous snapshot
-     * to compute a delta (e.g. the event loop and storage backend signals read the last entry to measure change since
-     * it).
-     */
-    getAll() {
-        return this.#snapshots;
-    }
-    /**
-     * Discards every retained snapshot. The built-in signals do this when they *start*, so that a session neither
-     * samples nor diffs against measurements from before the preceding downtime — pruning is relative to the newest
-     * snapshot rather than the wall clock, so stale entries would otherwise survive indefinitely. Clearing on start
-     * rather than on stop leaves a finished session readable.
-     */
-    clear() {
-        this.#snapshots = [];
-    }
-}
-/**
- * Evaluate whether a sample of `LoadSnapshot` values exceeds the given
- * overloaded ratio, using a time-weighted average. This is the shared
- * evaluation logic used by `SystemStatus` for all signal types.
- * @internal
- */
-export function evaluateLoadSignalSample(sample, overloadedRatio) {
-    if (sample.length === 0) {
-        return {
-            isOverloaded: false,
-            limitRatio: overloadedRatio,
-            actualRatio: 0,
-        };
-    }
-    const weights = [];
-    const values = [];
-    for (let i = 1; i < sample.length; i++) {
-        const previous = sample[i - 1];
-        const current = sample[i];
-        const weight = +current.createdAt - +previous.createdAt;
-        weights.push(weight || 1); // Prevent errors from 0ms long intervals (sync) between snapshots.
-        values.push(+current.isOverloaded);
-    }
-    const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);
-    return {
-        isOverloaded: wAvg > overloadedRatio,
-        limitRatio: overloadedRatio,
-        actualRatio: Math.round(wAvg * 1000) / 1000,
-    };
-}

package/autoscaling/memory_load_signal.d.ts

@@ -1,47 +0,0 @@
-import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
-import type { SystemInfo } from './system_status.js';
-/**
- * A snapshot produced by the built-in memory signal.
- * @internal
- */
-export interface MemorySnapshot extends LoadSnapshot {
-    usedBytes?: number;
-}
-/**
- * Tuning for the built-in **memory** load signal, as accepted both by {@link MemoryLoadSignal} and by the
- * {@link LoadSignalsOptions.memory|`memory`} shorthand on {@link LoadSignalsOptions}.
- */
-export interface MemoryLoadSignalOptions {
-    /**
-     * Defines the maximum ratio of total memory that can be used.
-     * Exceeding this limit overloads the memory.
-     * @default 0.9
-     */
-    maxUsedRatio?: number;
-    /**
-     * Maximum ratio of overloaded snapshots in a sample before memory counts as overloaded.
-     * @default 0.2
-     */
-    overloadedRatio?: number;
-}
-/**
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
- * threshold. Also warns when memory use becomes critical.
- *
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
- *
- * @category Scaling
- */
-export declare class MemoryLoadSignal implements LoadSignal {
-    #private;
-    readonly name = "memInfo";
-    readonly overloadedRatio: number;
-    constructor(options?: MemoryLoadSignalOptions);
-    start(context: LoadSignalStartContext): Promise<void>;
-    stop(): Promise<void>;
-    getSample(sampleDurationMillis?: number): LoadSnapshot[];
-    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
-    handle(systemInfo: SystemInfo): void;
-    private memoryOverloadWarning;
-    private getTotalMemoryBytes;
-}

package/autoscaling/memory_load_signal.js

@@ -1,106 +0,0 @@
-import { EventType } from '../events/event_manager.js';
-import { serviceLocator } from '../service_locator.js';
-import { getMemoryInfo } from '../system-info/memory-info.js';
-import { isContainerized } from '../system-info/runtime.js';
-import { SnapshotStore } from './load_signal.js';
-const RESERVE_MEMORY_RATIO = 0.5;
-const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
-/**
- * Tracks memory usage via `SYSTEM_INFO` events and reports overload when the used-to-available memory ratio exceeds a
- * threshold. Also warns when memory use becomes critical.
- *
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
- *
- * @category Scaling
- */
-export class MemoryLoadSignal {
-    name = 'memInfo';
-    overloadedRatio;
-    #store = new SnapshotStore();
-    #maxUsedRatio;
-    /** All resolved in `start()`, before anything that reads them can fire. */
-    #config;
-    #log;
-    #maxMemoryBytes;
-    #events;
-    #maxMemoryRatio;
-    #lastLoggedCriticalMemoryOverloadAt = null;
-    constructor(options = {}) {
-        this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;
-        this.overloadedRatio = options.overloadedRatio ?? 0.2;
-        this.handle = this.handle.bind(this);
-    }
-    async start(context) {
-        this.#store.useSampleWindow(context.maxSampleWindowMillis);
-        // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
-        this.#store.clear();
-        // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
-        // between systems) must not capture whichever services happened to be registered at that moment.
-        this.#config = serviceLocator.getConfiguration();
-        this.#events = serviceLocator.getEventManager();
-        this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
-        const memoryMbytes = this.#config.memoryMbytes ?? 0;
-        if (memoryMbytes > 0) {
-            this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;
-        }
-        else {
-            this.#maxMemoryRatio = this.#config.availableMemoryRatio;
-            if (!this.#maxMemoryRatio) {
-                throw new Error('availableMemoryRatio is not set in configuration.');
-            }
-            else {
-                this.#log.debug(`Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +
-                    'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
-            }
-            // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
-            this.#maxMemoryBytes = await this.getTotalMemoryBytes();
-        }
-        this.#events.on(EventType.SYSTEM_INFO, this.handle);
-    }
-    async stop() {
-        this.#events?.off(EventType.SYSTEM_INFO, this.handle);
-        this.#events = undefined;
-    }
-    getSample(sampleDurationMillis) {
-        return this.#store.getSample(sampleDurationMillis);
-    }
-    /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
-    handle(systemInfo) {
-        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        const { memCurrentBytes, memTotalBytes } = systemInfo;
-        let maxMemoryBytes = this.#maxMemoryBytes;
-        if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {
-            maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);
-        }
-        const snapshot = {
-            createdAt,
-            isOverloaded: memCurrentBytes / maxMemoryBytes > this.#maxUsedRatio,
-            usedBytes: memCurrentBytes,
-        };
-        this.#store.push(snapshot, createdAt);
-        this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
-    }
-    memoryOverloadWarning(systemInfo, maxMemoryBytes) {
-        const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
-        const { memCurrentBytes } = systemInfo;
-        const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        if (this.#lastLoggedCriticalMemoryOverloadAt &&
-            +createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
-            return;
-        const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;
-        const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;
-        const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
-        const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
-        if (isCriticalOverload) {
-            const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
-            const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
-            this.#log.warning('Memory is critically overloaded. ' +
-                `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
-            this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
-        }
-    }
-    async getTotalMemoryBytes() {
-        const containerized = this.#config.containerized ?? (await isContainerized());
-        return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
-    }
-}

package/autoscaling/snapshotter.d.ts

@@ -1,84 +0,0 @@
-import type { CpuLoadSignalOptions } from './cpu_load_signal.js';
-import type { EventLoopLoadSignalOptions } from './event_loop_load_signal.js';
-import type { LoadSignal, LoadSignalStartContext } from './load_signal.js';
-import type { MemoryLoadSignalOptions } from './memory_load_signal.js';
-import type { StorageBackendLoadSignalOptions } from './storage_backend_load_signal.js';
-/**
- * The load signals a {@link ConcurrencySystem} watches to decide whether the machine is overloaded.
- *
- * Each of the four built-in signals is configured by passing its options bag — shorthand for constructing the
- * corresponding {@link LoadSignal} class yourself, so `{ cpu: { overloadedRatio: 0.5 } }` is exactly
- * `{ cpu: false, custom: [new CpuLoadSignal({ overloadedRatio: 0.5 })] }`. Pass `false` to leave a resource
- * unwatched, and put anything else you want taken into account in {@link LoadSignalsOptions.custom|`custom`}.
- *
- * How far back the signals are evaluated is *not* set here, but by
- * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`} and
- * {@link ConcurrencySystemOptions.currentHistorySecs|`currentHistorySecs`}.
- */
-export interface LoadSignalsOptions {
-    /**
-     * Tuning for the built-in {@link MemoryLoadSignal} (used-memory limit + overload ratio), or `false` to switch
-     * it off.
-     */
-    memory?: MemoryLoadSignalOptions | false;
-    /**
-     * Tuning for the built-in {@link EventLoopLoadSignal} (snapshot interval + blocked-millis limit + overload
-     * ratio), or `false` to switch it off — which also stops its measuring interval.
-     */
-    eventLoop?: EventLoopLoadSignalOptions | false;
-    /**
-     * Tuning for the built-in {@link CpuLoadSignal} (overload ratio), or `false` to switch it off.
-     */
-    cpu?: CpuLoadSignalOptions | false;
-    /**
-     * Tuning for the built-in {@link StorageBackendLoadSignal} (snapshot interval + error limit + overload ratio),
-     * or `false` to switch it off — worth doing when the storage backend reports no rate-limit statistics, since the
-     * signal otherwise polls it every second to no purpose.
-     */
-    storageBackend?: StorageBackendLoadSignalOptions | false;
-    /**
-     * Additional {@link LoadSignal} implementations — e.g. navigation timeouts or proxy health — evaluated
-     * alongside the built-in four. If any signal reports overload, the system counts as overloaded. Their lifecycle
-     * is driven by the {@link ConcurrencySystem} they are given to, and their {@link LoadSignal.name|names} must
-     * not collide with an enabled built-in's.
-     */
-    custom?: LoadSignal[];
-}
-/**
- * An implementation detail of the {@link ConcurrencySystem}: the built-in signal tuning from
- * {@link LoadSignalsOptions}, minus the custom signals, which the system evaluates itself rather than collecting
- * them here.
- * @internal
- */
-export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
-/**
- * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
- * the ones that were not switched off and driving their shared lifecycle.
- *
- * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
- * are simply forwarded to the corresponding constructor.
- * @internal
- */
-export declare class Snapshotter {
-    #private;
-    /**
-     * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
-     * instances. Signals switched off through the options are simply absent — the system status reports them as
-     * not overloaded.
-     */
-    getLoadSignals(): LoadSignal[];
-    /**
-     * @param [options] All `Snapshotter` configuration options.
-     */
-    constructor(options?: SnapshotterOptions);
-    /**
-     * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
-     * be queried with, which is also how much history they retain.
-     */
-    start(context: LoadSignalStartContext): Promise<void>;
-    /**
-     * Stops all resource capturing.
-     */
-    stop(): Promise<void>;
-}

package/autoscaling/snapshotter.js

@@ -1,67 +0,0 @@
-import { CpuLoadSignal } from './cpu_load_signal.js';
-import { EventLoopLoadSignal } from './event_loop_load_signal.js';
-import { MemoryLoadSignal } from './memory_load_signal.js';
-import { StorageBackendLoadSignal } from './storage_backend_load_signal.js';
-/**
- * Owns the four built-in {@link LoadSignal} instances — {@link MemoryLoadSignal},
- * {@link EventLoopLoadSignal}, {@link CpuLoadSignal} and {@link StorageBackendLoadSignal} — constructing
- * the ones that were not switched off and driving their shared lifecycle.
- *
- * Configured indirectly through {@link ConcurrencySystemOptions.loadSignals|`loadSignals`}, whose per-signal bags
- * are simply forwarded to the corresponding constructor.
- * @internal
- */
-export class Snapshotter {
-    // Absent when switched off through the corresponding option (e.g. `storageBackend: false`).
-    #memorySignal;
-    #eventLoopSignal;
-    #cpuSignal;
-    #storageBackendSignal;
-    /**
-     * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
-     * instances. Signals switched off through the options are simply absent — the system status reports them as
-     * not overloaded.
-     */
-    getLoadSignals() {
-        const builtin = [
-            this.#memorySignal,
-            this.#eventLoopSignal,
-            this.#cpuSignal,
-            this.#storageBackendSignal,
-        ];
-        return builtin.filter((signal) => signal !== undefined);
-    }
-    /**
-     * @param [options] All `Snapshotter` configuration options.
-     */
-    constructor(options = {}) {
-        const { memory = {}, eventLoop = {}, cpu = {}, storageBackend = {} } = options;
-        // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
-        // over then too - so there is nothing to thread in here beyond the caller's tuning.
-        if (memory !== false)
-            this.#memorySignal = new MemoryLoadSignal(memory);
-        if (eventLoop !== false)
-            this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
-        if (cpu !== false)
-            this.#cpuSignal = new CpuLoadSignal(cpu);
-        if (storageBackend !== false)
-            this.#storageBackendSignal = new StorageBackendLoadSignal(storageBackend);
-    }
-    /**
-     * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will
-     * be queried with, which is also how much history they retain.
-     */
-    async start(context) {
-        await Promise.all(this.getLoadSignals().map(async (signal) => signal.start(context)));
-    }
-    /**
-     * Stops all resource capturing.
-     */
-    async stop() {
-        await Promise.all(this.getLoadSignals().map(async (signal) => signal.stop()));
-        // Allow microtask queue to unwind before stop returns.
-        await new Promise((resolve) => {
-            setImmediate(resolve);
-        });
-    }
-}

package/autoscaling/storage_backend_load_signal.d.ts

@@ -1,56 +0,0 @@
-import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
-/**
- * A snapshot produced by the built-in storage backend (rate-limit) signal.
- * @internal
- */
-export interface StorageBackendSnapshot extends LoadSnapshot {
-    rateLimitErrorCount: number;
-}
-/**
- * Tuning for the built-in **storage backend** (rate-limit) load signal, as accepted both by
- * {@link StorageBackendLoadSignal} and by the
- * {@link LoadSignalsOptions.storageBackend|`storageBackend`} shorthand on {@link LoadSignalsOptions}.
- */
-export interface StorageBackendLoadSignalOptions {
-    /**
-     * Defines the interval of checking the current state of the storage backend, in seconds.
-     * @default 1
-     */
-    snapshotIntervalSecs?: number;
-    /**
-     * Defines the maximum number of new rate limit errors within the given interval.
-     * @default 3
-     */
-    maxErrors?: number;
-    /**
-     * Maximum ratio of overloaded snapshots in a sample before the storage backend counts as overloaded.
-     * @default 0.3
-     */
-    overloadedRatio?: number;
-}
-/**
- * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
- * exceeds a threshold.
- *
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
- *
- * Switch it off entirely ({@link LoadSignalsOptions.storageBackend|`storageBackend: false`}) if the storage backend
- * reports no rate-limit statistics, since it otherwise polls it every second to no purpose.
- *
- * @category Scaling
- */
-export declare class StorageBackendLoadSignal implements LoadSignal {
-    #private;
-    readonly name = "storageBackendInfo";
-    readonly overloadedRatio: number;
-    constructor(options?: StorageBackendLoadSignalOptions);
-    start(context: LoadSignalStartContext): Promise<void>;
-    stop(): Promise<void>;
-    getSample(sampleDurationMillis?: number): LoadSnapshot[];
-    /**
-     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
-     * previous one.
-     * @internal Also lets tests drive the measurement without waiting on a timer.
-     */
-    handle(intervalCallback: () => unknown): void;
-}

package/autoscaling/storage_backend_load_signal.js

@@ -1,73 +0,0 @@
-import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { serviceLocator } from '../service_locator.js';
-import { SnapshotStore } from './load_signal.js';
-const RATE_LIMIT_ERROR_RETRY_COUNT = 2;
-/**
- * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
- * exceeds a threshold.
- *
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
- *
- * Switch it off entirely ({@link LoadSignalsOptions.storageBackend|`storageBackend: false`}) if the storage backend
- * reports no rate-limit statistics, since it otherwise polls it every second to no purpose.
- *
- * @category Scaling
- */
-export class StorageBackendLoadSignal {
-    name = 'storageBackendInfo';
-    overloadedRatio;
-    #store = new SnapshotStore();
-    #intervalMillis;
-    #maxErrors;
-    #interval;
-    #storageBackend;
-    constructor(options = {}) {
-        this.overloadedRatio = options.overloadedRatio ?? 0.3;
-        this.#intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;
-        this.#maxErrors = options.maxErrors ?? 3;
-        this.handle = this.handle.bind(this);
-    }
-    async start(context) {
-        this.#store.useSampleWindow(context.maxSampleWindowMillis);
-        // A new session starts from a clean slate, or its first measurement diffs the error count against the previous
-        // session's — possibly against a different backend, since it is resolved afresh just below.
-        this.#store.clear();
-        // Resolved here rather than in the constructor, where asking for the backend would instantiate a default one
-        // as a side effect - long before the crawler that owns the run has had a chance to register its own.
-        this.#storageBackend = serviceLocator.getStorageBackend();
-        this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
-    }
-    async stop() {
-        if (this.#interval)
-            betterClearInterval(this.#interval);
-        this.#interval = undefined;
-        this.#storageBackend = undefined;
-    }
-    getSample(sampleDurationMillis) {
-        return this.#store.getSample(sampleDurationMillis);
-    }
-    /**
-     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
-     * previous one.
-     * @internal Also lets tests drive the measurement without waiting on a timer.
-     */
-    handle(intervalCallback) {
-        const now = new Date();
-        const allErrorCounts = this.#storageBackend?.stats?.rateLimitErrors ?? [];
-        const currentErrCount = allErrorCounts[RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
-        const snapshot = {
-            createdAt: now,
-            isOverloaded: false,
-            rateLimitErrorCount: currentErrCount,
-        };
-        const all = this.#store.getAll();
-        const previousSnapshot = all[all.length - 1];
-        if (previousSnapshot) {
-            const delta = currentErrCount - previousSnapshot.rateLimitErrorCount;
-            if (delta > this.#maxErrors)
-                snapshot.isOverloaded = true;
-        }
-        this.#store.push(snapshot, now);
-        intervalCallback();
-    }
-}