@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.81

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/autoscaling/autoscaled_pool.d.ts

@@ -1,7 +1,6 @@
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
+import type { CrawleeLogger } from '../log.js';
 import type { SnapshotterOptions } from './snapshotter.js';
-import type { SystemInfo, SystemStatusOptions } from './system_status.js';
+import type { SystemStatusOptions } from './system_status.js';
 export interface AutoscaledPoolOptions {
     /**
      * A function that performs an asynchronous resource-intensive task.
@@ -99,7 +98,7 @@ export interface AutoscaledPoolOptions {
      * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
      */
     maxTasksPerMinute?: number;
-    log?: Log;
+    log?: CrawleeLogger;
 }
 /**
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
@@ -150,7 +149,6 @@ export interface AutoscaledPoolOptions {
  * @category Scaling
  */
 export declare class AutoscaledPool {
-    private readonly config;
     private readonly log;
     private readonly desiredConcurrencyRatio;
     private readonly scaleUpStepRatio;
@@ -172,6 +170,8 @@ export declare class AutoscaledPool {
     private resolve;
     private reject;
     private snapshotter;
+    /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
+    private loadSignals;
     private systemStatus;
     private autoscaleInterval;
     private maybeRunInterval;
@@ -179,7 +179,7 @@ export declare class AutoscaledPool {
     private queryingIsFinished;
     private tasksDonePerSecondInterval?;
     private _tasksPerMinute;
-    constructor(options: AutoscaledPoolOptions, config?: Configuration);
+    constructor(options: AutoscaledPoolOptions);
     /**
      * Gets the minimum number of tasks running in parallel.
      */
@@ -262,39 +262,38 @@ export declare class AutoscaledPool {
      *
      * It doesn't allow multiple concurrent runs of this method.
      */
-    protected _maybeRunTask(intervalCallback?: () => void): Promise<void>;
+    private maybeRunTask;
     /**
      * Gets called every autoScaleIntervalSecs and evaluates the current system status.
      * If the system IS NOT overloaded and the settings allow it, it scales up.
      * If the system IS overloaded and the settings allow it, it scales down.
      */
-    protected _autoscale(intervalCallback: () => void): void;
+    private autoscale;
     /**
      * Scales the pool up by increasing
      * the desired concurrency by the scaleUpStepRatio.
      *
      * @param systemStatus for logging
      */
-    protected _scaleUp(systemStatus: SystemInfo): void;
+    private scaleUp;
     /**
      * Scales the pool down by decreasing
      * the desired concurrency by the scaleDownStepRatio.
      *
      * @param systemStatus for logging
      */
-    protected _scaleDown(systemStatus: SystemInfo): void;
+    private scaleDown;
     /**
      * If there are no running tasks and this.isFinishedFunction() returns true then closes
      * the pool and resolves the pool's promise returned by the run() method.
      *
      * It doesn't allow multiple concurrent runs of this method.
      */
-    protected _maybeFinish(): Promise<void>;
+    private maybeFinish;
     /**
      * Cleans up resources.
      */
-    protected _destroy(): Promise<void>;
-    protected _incrementTasksDonePerSecond(intervalCallback: () => void): void;
-    protected get _isOverMaxRequestLimit(): boolean;
+    private destroy;
+    private incrementTasksDonePerSecond;
+    private get isOverMaxRequestLimit();
 }
-//# sourceMappingURL=autoscaled_pool.d.ts.map

package/autoscaling/autoscaled_pool.js

@@ -1,9 +1,8 @@
 import ow from 'ow';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
 import { CriticalError } from '../errors.js';
-import { log as defaultLog } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
 import { Snapshotter } from './snapshotter.js';
 import { SystemStatus } from './system_status.js';
 /**
@@ -55,7 +54,6 @@ import { SystemStatus } from './system_status.js';
  * @category Scaling
  */
 export class AutoscaledPool {
-    config;
     log;
     // Configurable properties.
     desiredConcurrencyRatio;
@@ -79,6 +77,8 @@ export class AutoscaledPool {
     resolve = null;
     reject = null;
     snapshotter;
+    /** Additional SystemStatus loadSignals - tracked here for initialization and cleanup */
+    loadSignals;
     systemStatus;
     autoscaleInterval;
     maybeRunInterval;
@@ -86,8 +86,7 @@ export class AutoscaledPool {
     queryingIsFinished;
     tasksDonePerSecondInterval;
     _tasksPerMinute = Array.from({ length: 60 }, () => 0);
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        this.config = config;
+    constructor(options) {
         ow(options, ow.object.exactShape({
             runTaskFunction: ow.function,
             isFinishedFunction: ow.function,
@@ -107,7 +106,7 @@ export class AutoscaledPool {
             log: ow.optional.object,
             maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
         }));
-        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = defaultLog, maxTasksPerMinute = Infinity, } = options;
+        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, maybeRunIntervalSecs = 0.5, loggingIntervalSecs = 60, taskTimeoutSecs = 0, autoscaleIntervalSecs = 10, systemStatusOptions, snapshotterOptions, log = serviceLocator.getLogger(), maxTasksPerMinute = Infinity, } = options;
         this.log = log.child({ prefix: 'AutoscaledPool' });
         // Configurable properties.
         this.desiredConcurrencyRatio = desiredConcurrencyRatio;
@@ -129,19 +128,17 @@ export class AutoscaledPool {
         this.isStopped = false;
         this.resolve = null;
         this.reject = null;
-        this._autoscale = this._autoscale.bind(this);
-        this._maybeRunTask = this._maybeRunTask.bind(this);
-        this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
+        this.autoscale = this.autoscale.bind(this);
+        this.maybeRunTask = this.maybeRunTask.bind(this);
+        this.incrementTasksDonePerSecond = this.incrementTasksDonePerSecond.bind(this);
         // Create instances with correct options.
         const ssoCopy = { ...systemStatusOptions };
         ssoCopy.snapshotter ??= new Snapshotter({
             ...snapshotterOptions,
             log: this.log,
-            config: this.config,
-            client: this.config.getStorageClient(),
         });
-        ssoCopy.config ??= this.config;
         this.snapshotter = ssoCopy.snapshotter;
+        this.loadSignals = ssoCopy.loadSignals ?? [];
         this.systemStatus = new SystemStatus(ssoCopy);
     }
     /**
@@ -204,15 +201,16 @@ export class AutoscaledPool {
             this.reject = reject;
         });
         await this.snapshotter.start();
+        await Promise.all(this.loadSignals.map((s) => s.start()));
         // This interval checks the system status and updates the desired concurrency accordingly.
-        this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
+        this.autoscaleInterval = betterSetInterval(this.autoscale, this.autoscaleIntervalMillis);
         // This is here because if we scale down to let's say 1, then after each promise is finished
-        // this._maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
+        // this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
         // in the crawler getting stuck and even after scaling up it never triggers another promise.
-        this.maybeRunInterval = betterSetInterval(this._maybeRunTask, this.maybeRunIntervalMillis);
+        this.maybeRunInterval = betterSetInterval(this.maybeRunTask, this.maybeRunIntervalMillis);
         if (this.maxTasksPerMinute !== Infinity) {
             // Start the interval that resets the counter of tasks per minute.
-            this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);
+            this.tasksDonePerSecondInterval = betterSetInterval(this.incrementTasksDonePerSecond, 1000);
         }
         try {
             await poolPromise;
@@ -220,7 +218,7 @@ export class AutoscaledPool {
         finally {
             // If resolve is null, the pool is already destroyed.
             if (this.resolve)
-                await this._destroy();
+                await this.destroy();
         }
     }
     /**
@@ -238,7 +236,7 @@ export class AutoscaledPool {
         this.isStopped = true;
         if (this.resolve) {
             this.resolve();
-            await this._destroy();
+            await this.destroy();
         }
     }
     /**
@@ -258,14 +256,17 @@ export class AutoscaledPool {
         this.isStopped = true;
         await new Promise((resolve, reject) => {
             let timeout;
+            let interval;
             if (timeoutSecs) {
                 timeout = setTimeout(() => {
+                    // Clean up the polling interval to prevent it from leaking on timeout.
+                    clearInterval(interval);
                     const err = new Error("The pool's running tasks did not finish" +
                         `in ${timeoutSecs} secs after pool.pause() invocation.`);
                     reject(err);
                 }, timeoutSecs);
             }
-            const interval = setInterval(() => {
+            interval = setInterval(() => {
                 if (this._currentConcurrency <= 0) {
                     // Clean up timeout and interval to prevent process hanging.
                     if (timeout)
@@ -290,7 +291,7 @@ export class AutoscaledPool {
      * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
      */
     async notify() {
-        setImmediate(this._maybeRunTask);
+        setImmediate(this.maybeRunTask);
     }
     /**
      * Starts a new task
@@ -300,7 +301,7 @@ export class AutoscaledPool {
      *
      * It doesn't allow multiple concurrent runs of this method.
      */
-    async _maybeRunTask(intervalCallback) {
+    async maybeRunTask(intervalCallback) {
         this.log.perf('Attempting to run a task.');
         // Check if the function was invoked by the maybeRunInterval and use an empty function if not.
         const done = intervalCallback || (() => { });
@@ -351,12 +352,12 @@ export class AutoscaledPool {
             this.log.perf('Task will not run. No tasks are ready.');
             done();
             // No tasks could mean that we're finished with all tasks.
-            return this._maybeFinish();
+            return this.maybeFinish();
         }
         // - we have already reached the maximum tasks per minute
         // we need to check this *after* checking if a task is ready to prevent hanging the pool
         // for an extra minute if there are no more tasks
-        if (this._isOverMaxRequestLimit) {
+        if (this.isOverMaxRequestLimit) {
             this.log.perf('Task will not run. Maximum tasks per minute reached.');
             return done();
         }
@@ -366,7 +367,7 @@ export class AutoscaledPool {
             this._tasksPerMinute[0]++;
             // Try to run next task to build up concurrency,
             // but defer it so it doesn't create a cycle.
-            setImmediate(this._maybeRunTask);
+            setImmediate(this.maybeRunTask);
             // We need to restart interval here, so that it doesn't get blocked by a stalled task.
             done();
             // Execute the current task.
@@ -380,7 +381,7 @@ export class AutoscaledPool {
             this.log.perf('Task finished.');
             this._currentConcurrency--;
             // Run task after the previous one finished.
-            setImmediate(this._maybeRunTask);
+            setImmediate(this.maybeRunTask);
         }
         catch (e) {
             const err = e;
@@ -403,12 +404,12 @@ export class AutoscaledPool {
      * If the system IS NOT overloaded and the settings allow it, it scales up.
      * If the system IS overloaded and the settings allow it, it scales down.
      */
-    _autoscale(intervalCallback) {
+    autoscale(intervalCallback) {
         // Don't scale if paused.
         if (this.isStopped)
             return intervalCallback();
         // Don't scale if we've hit the maximum requests per minute
-        if (this._isOverMaxRequestLimit)
+        if (this.isOverMaxRequestLimit)
             return intervalCallback();
         // Only scale up if:
         // - system has not been overloaded lately.
@@ -420,14 +421,14 @@ export class AutoscaledPool {
         const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);
         const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
         if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
-            this._scaleUp(systemStatus);
+            this.scaleUp(systemStatus);
         // Always scale down if:
         // - the system has been overloaded lately.
         const isSystemOverloaded = !isSystemIdle;
         // - we're over min concurrency.
         const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;
         if (isSystemOverloaded && weAreNotAtMin)
-            this._scaleDown(systemStatus);
+            this.scaleDown(systemStatus);
         // On periodic intervals, print comprehensive log information
         if (this.loggingIntervalMillis > 0) {
             const now = Date.now();
@@ -452,7 +453,7 @@ export class AutoscaledPool {
      *
      * @param systemStatus for logging
      */
-    _scaleUp(systemStatus) {
+    scaleUp(systemStatus) {
         const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
         this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);
         this.log.debug('scaling up', {
@@ -467,7 +468,7 @@ export class AutoscaledPool {
      *
      * @param systemStatus for logging
      */
-    _scaleDown(systemStatus) {
+    scaleDown(systemStatus) {
         const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);
         this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);
         this.log.debug('scaling down', {
@@ -482,7 +483,7 @@ export class AutoscaledPool {
      *
      * It doesn't allow multiple concurrent runs of this method.
      */
-    async _maybeFinish() {
+    async maybeFinish() {
         if (this.queryingIsFinished)
             return;
         if (this._currentConcurrency > 0)
@@ -508,7 +509,7 @@ export class AutoscaledPool {
     /**
      * Cleans up resources.
      */
-    async _destroy() {
+    async destroy() {
         this.resolve = null;
         this.reject = null;
         betterClearInterval(this.autoscaleInterval);
@@ -517,17 +518,17 @@ export class AutoscaledPool {
             betterClearInterval(this.tasksDonePerSecondInterval);
         if (this.snapshotter)
             await this.snapshotter.stop();
+        await Promise.all(this.loadSignals.map((s) => s.stop()));
     }
-    _incrementTasksDonePerSecond(intervalCallback) {
+    incrementTasksDonePerSecond(intervalCallback) {
         this._tasksPerMinute.unshift(0);
         this._tasksPerMinute.pop();
         return intervalCallback();
     }
-    get _isOverMaxRequestLimit() {
+    get isOverMaxRequestLimit() {
         if (this.maxTasksPerMinute === Infinity) {
             return false;
         }
         return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
     }
 }
-//# sourceMappingURL=autoscaled_pool.js.map

package/autoscaling/client_load_signal.d.ts

@@ -0,0 +1,25 @@
+import type { StorageBackend } from '@crawlee/types';
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+export interface ClientSnapshot extends LoadSnapshot {
+    rateLimitErrorCount: number;
+}
+export interface ClientLoadSignalOptions {
+    client: StorageBackend;
+    clientSnapshotIntervalSecs?: number;
+    maxClientErrors?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+}
+/**
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
+ * and reports overload when the error delta exceeds a threshold.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createClientLoadSignal(options: ClientLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<ClientSnapshot>;
+    handle: (cb: () => unknown) => void;
+    getSample(sampleDurationMillis?: number): ClientSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type ClientLoadSignal = ReturnType<typeof createClientLoadSignal>;

package/autoscaling/client_load_signal.js

@@ -0,0 +1,36 @@
+import { SnapshotStore } from './load_signal.js';
+const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+/**
+ * Periodically checks the storage client for rate-limit errors (HTTP 429)
+ * and reports overload when the error delta exceeds a threshold.
+ */
+export function createClientLoadSignal(options) {
+    const maxClientErrors = options.maxClientErrors ?? 3;
+    const signal = SnapshotStore.fromInterval({
+        name: 'clientInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.3,
+        intervalMillis: (options.clientSnapshotIntervalSecs ?? 1) * 1000,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, intervalCallback) {
+            const now = new Date();
+            const allErrorCounts = options.client.stats?.rateLimitErrors ?? [];
+            const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+            const snapshot = {
+                createdAt: now,
+                isOverloaded: false,
+                rateLimitErrorCount: currentErrCount,
+            };
+            const all = store.getAll();
+            const previousSnapshot = all[all.length - 1];
+            if (previousSnapshot) {
+                const { rateLimitErrorCount } = previousSnapshot;
+                const delta = currentErrCount - rateLimitErrorCount;
+                if (delta > maxClientErrors)
+                    snapshot.isOverloaded = true;
+            }
+            store.push(snapshot, now);
+            intervalCallback();
+        },
+    });
+    return signal;
+}

package/autoscaling/cpu_load_signal.d.ts

@@ -0,0 +1,28 @@
+import type { Configuration } from '../configuration.js';
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+import type { SystemInfo } from './system_status.js';
+export interface CpuSnapshot extends LoadSnapshot {
+    usedRatio: number;
+    ticks?: {
+        idle: number;
+        total: number;
+    };
+}
+export interface CpuLoadSignalOptions {
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+    config: Configuration;
+}
+/**
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
+ * the platform or local OS metrics indicate the CPU is overloaded.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createCpuLoadSignal(options: CpuLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<CpuSnapshot>;
+    handle: (payload: SystemInfo) => void;
+    getSample(sampleDurationMillis?: number): CpuSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type CpuLoadSignal = ReturnType<typeof createCpuLoadSignal>;

package/autoscaling/cpu_load_signal.js

@@ -0,0 +1,24 @@
+import { serviceLocator } from '../service_locator.js';
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
+ * the platform or local OS metrics indicate the CPU is overloaded.
+ */
+export function createCpuLoadSignal(options) {
+    return SnapshotStore.fromEvent({
+        name: 'cpuInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.4,
+        events: serviceLocator.getEventManager(),
+        event: "systemInfo" /* EventType.SYSTEM_INFO */,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, systemInfo) {
+            const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
+            const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
+            store.push({
+                createdAt,
+                isOverloaded: isCpuOverloaded,
+                usedRatio: Math.ceil(cpuCurrentUsage / 100),
+            }, createdAt);
+        },
+    });
+}

package/autoscaling/event_loop_load_signal.d.ts

@@ -0,0 +1,23 @@
+import type { LoadSnapshot } from './load_signal.js';
+import { SnapshotStore } from './load_signal.js';
+export interface EventLoopSnapshot extends LoadSnapshot {
+    exceededMillis: number;
+}
+export interface EventLoopLoadSignalOptions {
+    eventLoopSnapshotIntervalSecs?: number;
+    maxBlockedMillis?: number;
+    overloadedRatio?: number;
+    snapshotHistoryMillis?: number;
+}
+/**
+ * Periodically measures event loop delay and reports overload when the
+ * delay exceeds a configured threshold.
+ */
+// @ts-ignore optional peer dependency or compatibility with es2022
+export declare function createEventLoopLoadSignal(options?: EventLoopLoadSignalOptions): Omit<import("./load_signal.js").LoadSignal, "getSample"> & {
+    store: SnapshotStore<EventLoopSnapshot>;
+    handle: (cb: () => unknown) => void;
+    getSample(sampleDurationMillis?: number): EventLoopSnapshot[];
+};
+/** @internal Return type for backward compat in Snapshotter facade */
+export type EventLoopLoadSignal = ReturnType<typeof createEventLoopLoadSignal>;

package/autoscaling/event_loop_load_signal.js

@@ -0,0 +1,35 @@
+import { SnapshotStore } from './load_signal.js';
+/**
+ * Periodically measures event loop delay and reports overload when the
+ * delay exceeds a configured threshold.
+ */
+export function createEventLoopLoadSignal(options = {}) {
+    const intervalMillis = (options.eventLoopSnapshotIntervalSecs ?? 0.5) * 1000;
+    const maxBlockedMillis = options.maxBlockedMillis ?? 50;
+    const signal = SnapshotStore.fromInterval({
+        name: 'eventLoopInfo',
+        overloadedRatio: options.overloadedRatio ?? 0.6,
+        intervalMillis,
+        snapshotHistoryMillis: options.snapshotHistoryMillis,
+        handler(store, intervalCallback) {
+            const now = new Date();
+            const snapshot = {
+                createdAt: now,
+                isOverloaded: false,
+                exceededMillis: 0,
+            };
+            const all = store.getAll();
+            const previousSnapshot = all[all.length - 1];
+            if (previousSnapshot) {
+                const { createdAt } = previousSnapshot;
+                const delta = now.getTime() - +createdAt - intervalMillis;
+                if (delta > maxBlockedMillis)
+                    snapshot.isOverloaded = true;
+                snapshot.exceededMillis = Math.max(delta - maxBlockedMillis, 0);
+            }
+            store.push(snapshot, now);
+            intervalCallback();
+        },
+    });
+    return signal;
+}

package/autoscaling/index.d.ts

@@ -1,4 +1,8 @@
 export * from './autoscaled_pool.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.d.ts.map

package/autoscaling/index.js

@@ -1,4 +1,8 @@
 export * from './autoscaled_pool.js';
+export * from './client_load_signal.js';
+export * from './cpu_load_signal.js';
+export * from './event_loop_load_signal.js';
+export * from './load_signal.js';
+export * from './memory_load_signal.js';
 export * from './snapshotter.js';
 export * from './system_status.js';
-//# sourceMappingURL=index.js.map