@crawlee/core 4.0.0-beta.88 → 4.0.0-beta.89

package/autoscaling/concurrency_system.d.ts

@@ -0,0 +1,283 @@
+import type { CrawleeLogger } from '../log.js';
+import type { LoadSignalsOptions } from './snapshotter.js';
+import type { SystemInfo } from './system_status.js';
+export interface ConcurrencySystemOptions {
+    /**
+     * The minimum number of tasks running in parallel.
+     *
+     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
+     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
+     * @default 1
+     */
+    minConcurrency?: number;
+    /**
+     * The maximum number of tasks running in parallel.
+     * @default 200
+     */
+    maxConcurrency?: number;
+    /**
+     * The desired number of tasks that should be running parallel on the start of the pool,
+     * if there is a large enough supply of them.
+     * By default, it is `minConcurrency`.
+     */
+    desiredConcurrency?: number;
+    /**
+     * Minimum level of desired concurrency to reach before more scaling up is allowed.
+     * @default 0.90
+     */
+    desiredConcurrencyRatio?: number;
+    /**
+     * Defines the fractional amount of desired concurrency to be added with each scaling up.
+     * The minimum scaling step is one.
+     * @default 0.05
+     */
+    scaleUpStepRatio?: number;
+    /**
+     * Defines the amount of desired concurrency to be subtracted with each scaling down.
+     * The minimum scaling step is one.
+     * @default 0.05
+     */
+    scaleDownStepRatio?: number;
+    /**
+     * Specifies a period in which the instance logs its state, in seconds.
+     * Set to `null` to disable periodic logging.
+     * @default 60
+     */
+    loggingIntervalSecs?: number | null;
+    /**
+     * Defines in seconds how often the system should attempt to adjust the desired concurrency
+     * based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
+     * We suggest using a value from 5 to 20.
+     * @default 10
+     */
+    autoscaleIntervalSecs?: number;
+    /**
+     * The signals that tell the system whether the machine is overloaded: per-resource tuning for the built-in four
+     * (memory, event loop, CPU, client) plus any {@link LoadSignalsOptions.custom|`custom`} implementations of
+     * your own. See {@link LoadSignalsOptions}.
+     */
+    loadSignals?: LoadSignalsOptions;
+    /**
+     * How far back the **autoscaling** decisions look, in seconds — the window the historical system status is
+     * evaluated over, and therefore how much history the signals retain (the memory cost of raising it).
+     * @default 30
+     */
+    snapshotHistorySecs?: number;
+    /**
+     * How far back the **task-gating** decision looks, in seconds — the window used to judge whether the system is
+     * overloaded *right now*, before dispatching one more task. Deliberately shorter than
+     * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`}, so that dispatch reacts to
+     * spikes quickly while scaling stays stable.
+     * @default 5
+     */
+    currentHistorySecs?: number;
+    /**
+     * The maximum number of tasks per minute the system can run.
+     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
+     */
+    maxTasksPerMinute?: number;
+    log?: CrawleeLogger;
+}
+/**
+ * Identifies *who* is asking a governor for capacity: one {@link AutoscaledPool}, or the crawler driving it. The
+ * same object is passed on every call a pool makes, so per-consumer state can be keyed off it or off its `id`.
+ * @category Scaling
+ */
+export interface ConcurrencyConsumer {
+    /** Process-unique and human-readable — a crawler's is its {@link BasicCrawlerOptions.id|`id`} option. */
+    readonly id: string;
+}
+/**
+ * The contract between an {@link AutoscaledPool} and its concurrency "governor" — the object that answers *is
+ * there free compute for one more task?* and tracks the budget that tasks are booked against.
+ * {@link ConcurrencySystem} is the canonical implementation; the interface lets alternate governors be substituted
+ * without depending on its internals.
+ *
+ * Every allocation method is told which {@link ConcurrencyConsumer|consumer} is asking, so an implementation can
+ * allocate per consumer. {@link ConcurrencySystem} does not: it serves whoever asks first, which can starve a pool
+ * that joins a saturated system late.
+ * @category Scaling
+ */
+export interface IConcurrencySystem {
+    /**
+     * The number of tasks that should currently be running in parallel, assuming a sufficient supply of them. How it
+     * is derived is up to the implementation, hence read-only here — but it must always be at least `1`, or a pool
+     * could never start the first task.
+     */
+    readonly desiredConcurrency: number;
+    /** The number of parallel tasks currently booked against this governor, regardless of which pool booked them. */
+    readonly currentConcurrency: number;
+    /**
+     * Whether the governor is ready to be booked against. {@link AutoscaledPool.run|`pool.run()`} refuses to run
+     * when this is `false`. An implementation with no startup lifecycle simply reports `true`.
+     */
+    readonly isRunning: boolean;
+    /**
+     * May **one more** task start right now, on behalf of `consumer`? A cheap pre-check the pool consults before
+     * querying task readiness.
+     *
+     * Must **not** enforce rate limits that only make sense for ready tasks (e.g. a per-minute task cap): the pool
+     * calls this before knowing whether any task is ready, so refusing here would stall an already-empty queue.
+     *
+     * Must also return `true` whenever `consumer` has nothing in flight of its own. A `false` sends that pool straight
+     * to its finished-check **without** consulting `isTaskReadyFunction`, so a governor that starves an idle pool can
+     * make its `run()` resolve while work is still pending. Tracking bookings per consumer answers that directly;
+     * {@link ConcurrencySystem}, which does not, instead never refuses while
+     * {@link IConcurrencySystem.currentConcurrency|`currentConcurrency`} is `0`.
+     */
+    hasCapacityForTask(consumer: ConcurrencyConsumer): boolean;
+    /**
+     * Books a task against the budget for `consumer`, returning `false` (without booking) when there is no room — the
+     * budget is spent, the consumer is over its share, or an implementation-specific rate limit was reached.
+     *
+     * Must be an *atomic* (synchronous) check-and-book: several pools may share one governor, and a check separated
+     * from the booking by an `await` lets two of them claim the last free slot at once.
+     */
+    tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
+    /** Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject). */
+    registerTaskEnd(consumer: ConcurrencyConsumer): void;
+}
+/**
+ * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
+ * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
+ * over time.
+ *
+ * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
+ * of letting each scale independently and oversubscribe the machine.
+ *
+ * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
+ * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
+ * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
+ * system down for every borrower.
+ * @category Scaling
+ */
+export declare class ConcurrencySystem implements IConcurrencySystem {
+    private readonly log;
+    private readonly desiredConcurrencyRatio;
+    private readonly scaleUpStepRatio;
+    private readonly scaleDownStepRatio;
+    private readonly loggingIntervalMillis;
+    private readonly autoscaleIntervalMillis;
+    private readonly maxTasksPerMinute;
+    private _minConcurrency;
+    private _maxConcurrency;
+    private _desiredConcurrency;
+    private _currentConcurrency;
+    private lastLoggingTime?;
+    private _tasksPerMinute;
+    private readonly snapshotter;
+    private readonly loadSignals;
+    private readonly systemStatus;
+    private autoscaleInterval?;
+    private tasksDonePerSecondInterval?;
+    /** Whether the snapshotter and autoscaling intervals are currently running. */
+    private running;
+    /** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
+    private startPromise?;
+    /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
+    private warnedAboutQueryWhileStopped;
+    constructor(options?: ConcurrencySystemOptions);
+    /**
+     * Gets the minimum number of tasks running in parallel.
+     */
+    get minConcurrency(): number;
+    /**
+     * Sets the minimum number of tasks running in parallel.
+     *
+     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
+     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
+     */
+    set minConcurrency(value: number);
+    /**
+     * Gets the maximum number of tasks running in parallel.
+     */
+    get maxConcurrency(): number;
+    /**
+     * Sets the maximum number of tasks running in parallel. Lowering it below the current
+     * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
+     * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
+     * limit as they settle).
+     */
+    set maxConcurrency(value: number);
+    /**
+     * Gets the desired concurrency for the system,
+     * which is an estimated number of parallel tasks that the system can currently support.
+     */
+    get desiredConcurrency(): number;
+    /**
+     * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
+     * in parallel if there's large enough supply of tasks.
+     */
+    set desiredConcurrency(value: number);
+    /**
+     * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
+     * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
+     * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
+     * that is the limit callers set in order to protect something.
+     */
+    private clampDesiredConcurrency;
+    get currentConcurrency(): number;
+    /** Whether the system is currently monitoring load and autoscaling the budget. */
+    get isRunning(): boolean;
+    /**
+     * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
+     * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
+     * signal fails to start.
+     */
+    start(): Promise<void>;
+    private boot;
+    /**
+     * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
+     */
+    stop(): Promise<void>;
+    private shutDown;
+    /**
+     * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
+     * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
+     * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
+     * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
+     */
+    private warnIfNotRunning;
+    /**
+     * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
+     * reached) or when the machine is overloaded past `minConcurrency`.
+     *
+     * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
+     * interface, letting the answer be queried directly.
+     */
+    hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
+    /** Whether the per-minute task cap has been reached. */
+    private get isOverMaxRequestLimit();
+    /**
+     * Atomically books a task against the shared budget: re-checks
+     * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
+     * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
+     * room. Call right before the task actually runs.
+     *
+     * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
+     * extra minute.
+     */
+    tryRegisterTaskStart(consumer?: ConcurrencyConsumer): boolean;
+    /** Returns a slot to the shared budget, whoever booked it. */
+    registerTaskEnd(_consumer?: ConcurrencyConsumer): void;
+    /**
+     * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
+     * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
+     * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
+     */
+    getCurrentStatus(): SystemInfo;
+    /**
+     * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
+     * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
+     */
+    private _autoscale;
+    /**
+     * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
+     */
+    private _scaleUp;
+    /**
+     * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
+     */
+    private _scaleDown;
+    private _incrementTasksDonePerSecond;
+}

package/autoscaling/concurrency_system.js

@@ -0,0 +1,350 @@
+import ow from 'ow';
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service_locator.js';
+import { Snapshotter } from './snapshotter.js';
+import { SystemStatus } from './system_status.js';
+/**
+ * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
+ * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
+ * over time.
+ *
+ * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
+ * of letting each scale independently and oversubscribe the machine.
+ *
+ * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
+ * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
+ * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
+ * system down for every borrower.
+ * @category Scaling
+ */
+export class ConcurrencySystem {
+    log;
+    desiredConcurrencyRatio;
+    scaleUpStepRatio;
+    scaleDownStepRatio;
+    loggingIntervalMillis;
+    autoscaleIntervalMillis;
+    maxTasksPerMinute;
+    _minConcurrency;
+    _maxConcurrency;
+    _desiredConcurrency;
+    _currentConcurrency = 0;
+    lastLoggingTime;
+    _tasksPerMinute = Array.from({ length: 60 }, () => 0);
+    snapshotter;
+    loadSignals;
+    systemStatus;
+    autoscaleInterval;
+    tasksDonePerSecondInterval;
+    /** Whether the snapshotter and autoscaling intervals are currently running. */
+    running = false;
+    /** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
+    startPromise;
+    /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
+    warnedAboutQueryWhileStopped = false;
+    constructor(options = {}) {
+        ow(options, ow.object.exactShape({
+            maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
+            minConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
+            desiredConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
+            desiredConcurrencyRatio: ow.optional.number.greaterThan(0).lessThan(1),
+            scaleUpStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
+            scaleDownStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
+            loggingIntervalSecs: ow.any(ow.number.greaterThan(0), ow.nullOrUndefined),
+            autoscaleIntervalSecs: ow.optional.number.greaterThan(0),
+            loadSignals: ow.optional.object,
+            snapshotHistorySecs: ow.optional.number.greaterThan(0),
+            currentHistorySecs: ow.optional.number.greaterThan(0),
+            log: ow.optional.object,
+            maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
+        }));
+        const { maxConcurrency = 200, minConcurrency = 1, desiredConcurrency, desiredConcurrencyRatio = 0.9, scaleUpStepRatio = 0.05, scaleDownStepRatio = 0.05, loggingIntervalSecs = 60, autoscaleIntervalSecs = 10, loadSignals = {}, snapshotHistorySecs, currentHistorySecs, log = serviceLocator.getLogger(), maxTasksPerMinute = Infinity, } = options;
+        this.log = log.child({ prefix: 'ConcurrencySystem' });
+        this.desiredConcurrencyRatio = desiredConcurrencyRatio;
+        this.scaleUpStepRatio = scaleUpStepRatio;
+        this.scaleDownStepRatio = scaleDownStepRatio;
+        this.loggingIntervalMillis = (loggingIntervalSecs ?? 0) * 1000;
+        this.autoscaleIntervalMillis = autoscaleIntervalSecs * 1000;
+        this.maxTasksPerMinute = maxTasksPerMinute;
+        this._minConcurrency = minConcurrency;
+        this._maxConcurrency = maxConcurrency;
+        this._desiredConcurrency = desiredConcurrency ?? minConcurrency;
+        this.clampDesiredConcurrency();
+        this._autoscale = this._autoscale.bind(this);
+        this._incrementTasksDonePerSecond = this._incrementTasksDonePerSecond.bind(this);
+        // The built-in signals are collected by the snapshotter; custom ones are simply evaluated alongside them.
+        const { custom: customLoadSignals = [], ...builtinSignalOptions } = loadSignals;
+        this.snapshotter = new Snapshotter(builtinSignalOptions);
+        this.loadSignals = customLoadSignals;
+        this.systemStatus = new SystemStatus({
+            snapshotter: this.snapshotter,
+            loadSignals: customLoadSignals,
+            currentHistorySecs,
+            // Both windows are requested from the signals explicitly, so a signal's own retention can neither widen
+            // nor (given the start context below) narrow what it contributes.
+            historySecs: snapshotHistorySecs,
+        });
+    }
+    /**
+     * Gets the minimum number of tasks running in parallel.
+     */
+    get minConcurrency() {
+        return this._minConcurrency;
+    }
+    /**
+     * Sets the minimum number of tasks running in parallel.
+     *
+     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
+     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
+     */
+    set minConcurrency(value) {
+        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        this._minConcurrency = value;
+        this.clampDesiredConcurrency();
+    }
+    /**
+     * Gets the maximum number of tasks running in parallel.
+     */
+    get maxConcurrency() {
+        return this._maxConcurrency;
+    }
+    /**
+     * Sets the maximum number of tasks running in parallel. Lowering it below the current
+     * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
+     * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
+     * limit as they settle).
+     */
+    set maxConcurrency(value) {
+        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        this._maxConcurrency = value;
+        this.clampDesiredConcurrency();
+    }
+    /**
+     * Gets the desired concurrency for the system,
+     * which is an estimated number of parallel tasks that the system can currently support.
+     */
+    get desiredConcurrency() {
+        return this._desiredConcurrency;
+    }
+    /**
+     * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
+     * in parallel if there's large enough supply of tasks.
+     */
+    set desiredConcurrency(value) {
+        ow(value, ow.optional.number.integer.greaterThanOrEqual(1));
+        this._desiredConcurrency = value;
+        this.clampDesiredConcurrency();
+    }
+    /**
+     * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
+     * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
+     * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
+     * that is the limit callers set in order to protect something.
+     */
+    clampDesiredConcurrency() {
+        const atLeastMin = Math.max(this._desiredConcurrency, this._minConcurrency);
+        this._desiredConcurrency = Math.min(atLeastMin, this._maxConcurrency);
+    }
+    get currentConcurrency() {
+        return this._currentConcurrency;
+    }
+    /** Whether the system is currently monitoring load and autoscaling the budget. */
+    get isRunning() {
+        return this.running;
+    }
+    /**
+     * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
+     * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
+     * signal fails to start.
+     */
+    async start() {
+        // Unwound and dropped again on failure, so a later `start()` retries instead of resolving instantly against
+        // a system that is down.
+        this.startPromise ??= this.boot().catch(async (error) => {
+            this.startPromise = undefined;
+            await this.shutDown();
+            throw error;
+        });
+        await this.startPromise;
+    }
+    async boot() {
+        // Per-session measurement state, reset so a restarted system isn't judged on the previous session. The
+        // per-minute window matters most: its ageing interval is cleared while we are down, so starts from before an
+        // arbitrarily long stop would otherwise still count against "this minute" and trip the cap immediately.
+        this._tasksPerMinute = Array.from({ length: 60 }, () => 0);
+        this.lastLoggingTime = undefined;
+        this.warnedAboutQueryWhileStopped = false;
+        // Signals are told how much history to keep when they start: exactly the longest window they will be sampled
+        // over, so nobody has to guess a retention value that matches this system's configuration.
+        const startContext = { maxSampleWindowMillis: this.systemStatus.maxSampleWindowMillis };
+        await this.snapshotter.start(startContext);
+        await Promise.all(this.loadSignals.map(async (s) => s.start(startContext)));
+        this.autoscaleInterval = betterSetInterval(this._autoscale, this.autoscaleIntervalMillis);
+        if (this.maxTasksPerMinute !== Infinity) {
+            this.tasksDonePerSecondInterval = betterSetInterval(this._incrementTasksDonePerSecond, 1000);
+        }
+        // Last, so `isRunning` never claims a system whose signals aren't collecting yet.
+        this.running = true;
+    }
+    /**
+     * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
+     */
+    async stop() {
+        if (this.startPromise === undefined) {
+            return;
+        }
+        // Waited out rather than interrupted, or the intervals a starting signal is about to register outlive us.
+        await this.startPromise.catch(() => { });
+        this.startPromise = undefined;
+        this.running = false;
+        await this.shutDown();
+    }
+    async shutDown() {
+        if (this.autoscaleInterval)
+            betterClearInterval(this.autoscaleInterval);
+        if (this.tasksDonePerSecondInterval)
+            betterClearInterval(this.tasksDonePerSecondInterval);
+        await this.snapshotter.stop();
+        await Promise.all(this.loadSignals.map(async (s) => s.stop()));
+    }
+    /**
+     * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
+     * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
+     * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
+     * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
+     */
+    warnIfNotRunning() {
+        if (this.running || this.warnedAboutQueryWhileStopped) {
+            return;
+        }
+        this.warnedAboutQueryWhileStopped = true;
+        this.log.warning('Capacity is being queried on a ConcurrencySystem that is not running, so system load is no longer being ' +
+            'monitored and the concurrency will no longer be adjusted. Whoever creates a ConcurrencySystem owns ' +
+            'its lifecycle: call `await concurrencySystem.stop()` only once every pool and crawler borrowing it ' +
+            'has finished.');
+    }
+    /**
+     * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
+     * reached) or when the machine is overloaded past `minConcurrency`.
+     *
+     * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
+     * interface, letting the answer be queried directly.
+     */
+    hasCapacityForTask(_consumer) {
+        this.warnIfNotRunning();
+        if (this._currentConcurrency >= this._desiredConcurrency) {
+            this.log.perf('Task will not run. Desired concurrency achieved.');
+            return false;
+        }
+        const currentStatus = this.systemStatus.getCurrentStatus();
+        const { isSystemIdle } = currentStatus;
+        if (!isSystemIdle && this._currentConcurrency >= this._minConcurrency) {
+            this.log.perf('Task will not be run. System is overloaded.', currentStatus);
+            return false;
+        }
+        return true;
+    }
+    /** Whether the per-minute task cap has been reached. */
+    get isOverMaxRequestLimit() {
+        if (this.maxTasksPerMinute === Infinity) {
+            return false;
+        }
+        return this._tasksPerMinute.reduce((acc, curr) => acc + curr, 0) >= this.maxTasksPerMinute;
+    }
+    /**
+     * Atomically books a task against the shared budget: re-checks
+     * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
+     * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
+     * room. Call right before the task actually runs.
+     *
+     * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
+     * extra minute.
+     */
+    tryRegisterTaskStart(consumer) {
+        if (!this.hasCapacityForTask(consumer)) {
+            return false;
+        }
+        if (this.isOverMaxRequestLimit) {
+            this.log.perf('Task will not run. Maximum tasks per minute reached.');
+            return false;
+        }
+        this._currentConcurrency++;
+        this._tasksPerMinute[0]++;
+        return true;
+    }
+    /** Returns a slot to the shared budget, whoever booked it. */
+    registerTaskEnd(_consumer) {
+        this._currentConcurrency--;
+    }
+    /**
+     * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
+     * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
+     * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
+     */
+    getCurrentStatus() {
+        return this.systemStatus.getCurrentStatus();
+    }
+    /**
+     * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
+     * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
+     */
+    _autoscale(intervalCallback) {
+        if (this.isOverMaxRequestLimit)
+            return intervalCallback();
+        const systemStatus = this.systemStatus.getHistoricalStatus();
+        const { isSystemIdle } = systemStatus;
+        const weAreNotAtMax = this._desiredConcurrency < this._maxConcurrency;
+        const minCurrentConcurrency = Math.floor(this._desiredConcurrency * this.desiredConcurrencyRatio);
+        const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
+        if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
+            this._scaleUp(systemStatus);
+        const isSystemOverloaded = !isSystemIdle;
+        const weAreNotAtMin = this._desiredConcurrency > this._minConcurrency;
+        if (isSystemOverloaded && weAreNotAtMin)
+            this._scaleDown(systemStatus);
+        if (this.loggingIntervalMillis > 0) {
+            const now = Date.now();
+            if (this.lastLoggingTime == null) {
+                this.lastLoggingTime = now;
+            }
+            else if (now > this.lastLoggingTime + this.loggingIntervalMillis) {
+                this.lastLoggingTime = now;
+                this.log.info('state', {
+                    currentConcurrency: this._currentConcurrency,
+                    desiredConcurrency: this._desiredConcurrency,
+                    systemStatus,
+                });
+            }
+        }
+        return intervalCallback();
+    }
+    /**
+     * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
+     */
+    _scaleUp(systemStatus) {
+        const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
+        this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);
+        this.log.debug('scaling up', {
+            oldConcurrency: this._desiredConcurrency - step,
+            newConcurrency: this._desiredConcurrency,
+            systemStatus,
+        });
+    }
+    /**
+     * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
+     */
+    _scaleDown(systemStatus) {
+        const step = Math.ceil(this._desiredConcurrency * this.scaleDownStepRatio);
+        this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);
+        this.log.debug('scaling down', {
+            oldConcurrency: this._desiredConcurrency + step,
+            newConcurrency: this._desiredConcurrency,
+            systemStatus,
+        });
+    }
+    _incrementTasksDonePerSecond(intervalCallback) {
+        this._tasksPerMinute.unshift(0);
+        this._tasksPerMinute.pop();
+        return intervalCallback();
+    }
+}