@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/autoscaling/autoscaled_pool.js

@@ -1,386 +0,0 @@
-import { z } from 'zod';
-import { addTimeoutToPromise } from '@apify/timeout';
-import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { CriticalError } from '../errors.js';
-import { serviceLocator } from '../service_locator.js';
-import { parseArgument, schemas, validators } from '../validators.js';
-// `schemas.anyObject` and `objectWithKeys`-based validators pass values through by reference
-// (object schemas return a pruned plain copy), so class instances like loggers and the
-// concurrency system keep their prototype.
-const autoscaledPoolOptionsSchema = z.strictObject({
-    runTaskFunction: schemas.anyFunction,
-    isFinishedFunction: schemas.anyFunction,
-    isTaskReadyFunction: schemas.anyFunction,
-    maybeRunIntervalSecs: schemas.anyNumber
-        .refine((value) => value > 0, 'Expected a number greater than 0')
-        .default(0.5),
-    taskTimeoutSecs: schemas.anyNumber
-        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
-        .default(0),
-    log: validators.logger.default(() => serviceLocator.getLogger()),
-    concurrencySystem: schemas.anyObject,
-    consumer: schemas.anyObject.refine((value) => typeof value.id === 'string' && value.id.length > 0, "Expected an object with a non-empty string 'id'"),
-});
-/**
- * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.
- * The pool only starts new tasks while its {@link IConcurrencySystem|concurrency system} reports free capacity —
- * that governor is what monitors CPU, memory and event loop load and autoscales the concurrency budget.
- *
- * Before running the pool, you need to implement the following three functions:
- * {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`},
- * {@link TaskLoopPredicates.isTaskReadyFunction|`isTaskReadyFunction`} and
- * {@link TaskLoopPredicates.isFinishedFunction|`isFinishedFunction`}.
- *
- * The auto-scaled pool is started by calling the {@link AutoscaledPool.run} function.
- * The pool periodically queries `isTaskReadyFunction` for more tasks, managing optimal concurrency, until the function
- * resolves to `false`. The pool then queries `isFinishedFunction`. If it resolves to `true`, the run finishes after all
- * running tasks complete. If it resolves to `false`, it assumes there will be more tasks available later and keeps
- * periodically querying for tasks.
- * If any of the tasks throws then the {@link AutoscaledPool.run} function rejects the promise with an error.
- *
- * The pool evaluates whether it should start a new task every time one of the tasks finishes
- * and also in the interval set by the `options.maybeRunIntervalSecs` parameter.
- *
- * **Example usage:**
- *
- * ```javascript
- * const concurrencySystem = new ConcurrencySystem({ maxConcurrency: 50 });
- * await concurrencySystem.start();
- *
- * const pool = new AutoscaledPool({
- *     concurrencySystem,
- *     consumer: { id: 'my-pool' },
- *     runTaskFunction: async () => {
- *         // Run some resource-intensive asynchronous operation here.
- *     },
- *     isTaskReadyFunction: async () => {
- *         // Tell the pool whether more tasks are ready to be processed.
- *         // Return true or false
- *     },
- *     isFinishedFunction: async () => {
- *         // Tell the pool whether it should finish
- *         // or wait for more tasks to become available.
- *         // Return true or false
- *     }
- * });
- *
- * try {
- *     await pool.run();
- * } finally {
- *     await concurrencySystem.stop();
- * }
- * ```
- *
- * @internal
- */
-export class AutoscaledPool {
-    #log;
-    // Configurable properties.
-    #maybeRunIntervalMillis;
-    #taskTimeoutMillis;
-    #runTaskFunction;
-    #isFinishedFunction;
-    #isTaskReadyFunction;
-    #concurrencySystem;
-    #consumer;
-    // Internal properties.
-    #isStopped = false;
-    #resolve = null;
-    #reject = null;
-    #maybeRunInterval;
-    #queryingIsTaskReady;
-    #queryingIsFinished;
-    /**
-     * This pool's *own* in-flight task count, as opposed to {@link AutoscaledPool.currentConcurrency}, which is the
-     * (possibly shared) governor's total. `pause()` and `maybeFinish()` care only about this pool draining.
-     */
-    #ownConcurrency = 0;
-    constructor(options) {
-        const { runTaskFunction, isFinishedFunction, isTaskReadyFunction, maybeRunIntervalSecs, taskTimeoutSecs, log, concurrencySystem, consumer, } = parseArgument(options, autoscaledPoolOptionsSchema, 'AutoscaledPoolOptions');
-        this.#log = log.child({ prefix: 'AutoscaledPool' });
-        // Configurable properties.
-        this.#maybeRunIntervalMillis = maybeRunIntervalSecs * 1000;
-        this.#taskTimeoutMillis = taskTimeoutSecs * 1000;
-        this.#runTaskFunction = runTaskFunction;
-        this.#isFinishedFunction = isFinishedFunction;
-        this.#isTaskReadyFunction = isTaskReadyFunction;
-        this.#concurrencySystem = concurrencySystem;
-        this.#consumer = consumer;
-        // Internal properties.
-        this.#isStopped = false;
-        this.#resolve = null;
-        this.#reject = null;
-        this.maybeRunTask = this.maybeRunTask.bind(this);
-    }
-    /**
-     * The governor backing this pool, as supplied to the constructor — exposed as the read-only
-     * {@link IConcurrencySystem} contract.
-     *
-     * This and the two getters below are telemetry only: concurrency is configured and tuned on the concrete
-     * {@link ConcurrencySystem} its owner holds, never through the pool.
-     */
-    get system() {
-        return this.#concurrencySystem;
-    }
-    /** The estimated number of parallel tasks the governor can currently support. */
-    get desiredConcurrency() {
-        return this.#concurrencySystem.desiredConcurrency;
-    }
-    /**
-     * The number of parallel tasks currently booked against the governor. When it is shared, this counts every
-     * borrowing pool's tasks, not just this one's.
-     */
-    get currentConcurrency() {
-        return this.#concurrencySystem.currentConcurrency;
-    }
-    /**
-     * Runs the auto-scaled pool. Returns a promise that gets resolved or rejected once
-     * all the tasks are finished or one of them fails.
-     *
-     * Throws if the {@link IConcurrencySystem|concurrency system} it borrows was never started — the pool assumes
-     * a running governor and cannot start one it does not own.
-     */
-    async run() {
-        // Checked here, on an awaited path — the capacity queries inside the task loop run from intervals and
-        // `setImmediate`, where a throw would become an unhandled rejection and hang `run()` forever.
-        if (!this.#concurrencySystem.isRunning) {
-            throw new CriticalError('The ConcurrencySystem this AutoscaledPool borrows has not been started, so system load would not be ' +
-                'monitored and the concurrency would never be adjusted. Whoever creates a ConcurrencySystem owns ' +
-                'its lifecycle: call `await concurrencySystem.start()` before running the pools or crawlers that ' +
-                'use it, and `await concurrencySystem.stop()` once they are all done.');
-        }
-        const poolPromise = new Promise((resolve, reject) => {
-            this.#resolve = resolve;
-            this.#reject = reject;
-        });
-        // This is here because if we scale down to let's say 1, then after each promise is finished
-        // this.maybeRunTask() doesn't trigger another one. So if that 1 instance gets stuck it results
-        // in the crawler getting stuck and even after scaling up it never triggers another promise.
-        this.#maybeRunInterval = betterSetInterval(this.maybeRunTask, this.#maybeRunIntervalMillis);
-        try {
-            await poolPromise;
-        }
-        finally {
-            // If resolve is null, the pool is already destroyed.
-            if (this.#resolve)
-                await this.destroy();
-        }
-    }
-    /**
-     * Aborts the run of the auto-scaled pool and destroys it. The promise returned from
-     * the {@link AutoscaledPool.run} function will immediately resolve, no more new tasks
-     * will be spawned and all running tasks will be left in their current state.
-     *
-     * Due to the nature of the tasks, auto-scaled pool cannot reliably guarantee abortion
-     * of all the running tasks, therefore, no abortion is attempted and some of the tasks
-     * may finish, while others may not. Essentially, auto-scaled pool doesn't care about
-     * their state after the invocation of `.abort()`, but that does not mean that some
-     * parts of their asynchronous chains of commands will not execute.
-     */
-    async abort() {
-        this.#isStopped = true;
-        if (this.#resolve) {
-            this.#resolve();
-            await this.destroy();
-        }
-    }
-    /**
-     * Prevents the auto-scaled pool from starting new tasks, but allows the running ones to finish
-     * (unlike abort, which terminates them). Used together with {@link AutoscaledPool.resume}
-     *
-     * The function's promise will resolve once all running tasks have completed and the pool
-     * is effectively idle. If the `timeoutSecs` argument is provided, the promise will reject
-     * with a timeout error after the `timeoutSecs` seconds.
-     *
-     * The promise returned from the {@link AutoscaledPool.run} function will not resolve
-     * when `.pause()` is invoked (unlike abort, which resolves it).
-     *
-     * > *NOTE:* Pausing the pool does not suspend the (possibly shared) {@link ConcurrencySystem} — its
-     * autoscaling and resource monitoring keep running, since other pools borrowing it may still be active. To silence
-     * it during a long pause, its owner can `stop()` and `start()` it again.
-     */
-    async pause(timeoutSecs) {
-        if (this.#isStopped)
-            return;
-        this.#isStopped = true;
-        await new Promise((resolve, reject) => {
-            let timeout;
-            let interval;
-            if (timeoutSecs) {
-                timeout = setTimeout(() => {
-                    // Clean up the polling interval to prevent it from leaking on timeout.
-                    clearInterval(interval);
-                    const err = new Error("The pool's running tasks did not finish" +
-                        `in ${timeoutSecs} secs after pool.pause() invocation.`);
-                    reject(err);
-                }, timeoutSecs);
-            }
-            interval = setInterval(() => {
-                if (this.#ownConcurrency <= 0) {
-                    // Clean up timeout and interval to prevent process hanging.
-                    if (timeout)
-                        clearTimeout(timeout);
-                    clearInterval(interval);
-                    resolve();
-                }
-            }, this.#maybeRunIntervalMillis);
-        });
-    }
-    /**
-     * Resumes the operation of the autoscaled-pool by allowing more tasks to be run.
-     * Used together with {@link AutoscaledPool.pause}
-     *
-     * Tasks will automatically start running again in `options.maybeRunIntervalSecs`.
-     */
-    resume() {
-        this.#isStopped = false;
-    }
-    /**
-     * Explicitly check the queue for new tasks. The AutoscaledPool checks the queue for new tasks periodically,
-     * every `maybeRunIntervalSecs` seconds. If you want to trigger the processing immediately, use this method.
-     */
-    async notify() {
-        setImmediate(this.maybeRunTask);
-    }
-    /**
-     * Starts a new task
-     * if the number of running tasks (current concurrency) is lower than desired concurrency
-     * and the system is not currently overloaded
-     * and `isTaskReadyFunction()` returns true.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    async maybeRunTask(intervalCallback) {
-        this.#log.perf('Attempting to run a task.');
-        // Check if the function was invoked by the maybeRunInterval and use an empty function if not.
-        const done = intervalCallback || (() => { });
-        // Prevent starting a new task if:
-        // - the pool is paused or aborted
-        if (this.#isStopped) {
-            this.#log.perf('Task will not run. AutoscaledPool is stopped.');
-            return done();
-        }
-        // - we are already querying for a task.
-        if (this.#queryingIsTaskReady) {
-            this.#log.perf('Task will not run. Waiting for a ready task.');
-            return done();
-        }
-        // - the budget has room for us.
-        if (!this.#concurrencySystem.hasCapacityForTask(this.#consumer)) {
-            done();
-            // A shared governor's budget can stay saturated by another pool indefinitely, so we still have to be able
-            // to notice that *this* pool has run out of work — `maybeFinish()` is the only thing that ever resolves
-            // `run()`. It no-ops while this pool has tasks of its own in flight, which is every case in which an
-            // unshared governor reports no capacity.
-            return this.maybeFinish();
-        }
-        // - a task is ready.
-        this.#queryingIsTaskReady = true;
-        let isTaskReady;
-        try {
-            this.#log.perf('Checking for ready tasks.');
-            isTaskReady = await this.#isTaskReadyFunction();
-        }
-        catch (e) {
-            const err = e;
-            this.#log.perf('Checking for ready tasks failed.');
-            // We might have already rejected this promise.
-            if (this.#reject) {
-                // No need to log all concurrent errors.
-                this.#log.exception(err, 'isTaskReadyFunction failed');
-                this.#reject(err);
-            }
-        }
-        finally {
-            this.#queryingIsTaskReady = false;
-        }
-        if (!isTaskReady) {
-            this.#log.perf('Task will not run. No tasks are ready.');
-            done();
-            // No tasks could mean that we're finished with all tasks.
-            return this.maybeFinish();
-        }
-        // - the budget still has room. Re-checked atomically, because another pool sharing the governor may have taken
-        // the last free slot while we awaited `isTaskReadyFunction` above.
-        if (!this.#concurrencySystem.tryRegisterTaskStart(this.#consumer)) {
-            return done();
-        }
-        this.#ownConcurrency++;
-        try {
-            // Everything's fine. Run task.
-            // Try to run next task to build up concurrency,
-            // but defer it so it doesn't create a cycle.
-            setImmediate(this.maybeRunTask);
-            // We need to restart interval here, so that it doesn't get blocked by a stalled task.
-            done();
-            // Execute the current task.
-            this.#log.perf('Running a task.');
-            if (this.#taskTimeoutMillis > 0) {
-                await addTimeoutToPromise(async () => this.#runTaskFunction(), this.#taskTimeoutMillis, `runTaskFunction timed out after ${this.#taskTimeoutMillis / 1000} seconds.`);
-            }
-            else {
-                await this.#runTaskFunction();
-            }
-            this.#log.perf('Task finished.');
-            // Run task after the previous one finished. Only on success: a failed task rejects the pool, and
-            // nudging the loop afterwards could start work on an already destroyed pool.
-            setImmediate(this.maybeRunTask);
-        }
-        catch (e) {
-            const err = e;
-            this.#log.perf('Running a task failed.');
-            // We might have already rejected this promise.
-            if (this.#reject) {
-                // No need to log all concurrent errors.
-                if (
-                // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
-                !(e instanceof CriticalError)) {
-                    this.#log.exception(err, 'runTaskFunction failed.');
-                }
-                this.#reject(err);
-            }
-        }
-        finally {
-            this.#concurrencySystem.registerTaskEnd(this.#consumer);
-            this.#ownConcurrency--;
-        }
-        return undefined;
-    }
-    /**
-     * If there are no running tasks and `isFinishedFunction()` returns true then closes
-     * the pool and resolves the pool's promise returned by the run() method.
-     *
-     * It doesn't allow multiple concurrent runs of this method.
-     */
-    async maybeFinish() {
-        if (this.#queryingIsFinished)
-            return;
-        if (this.#ownConcurrency > 0)
-            return;
-        this.#queryingIsFinished = true;
-        try {
-            const isFinished = await this.#isFinishedFunction();
-            if (isFinished && this.#resolve)
-                this.#resolve();
-        }
-        catch (e) {
-            const err = e;
-            if (this.#reject) {
-                // No need to log all concurrent errors.
-                this.#log.exception(err, 'isFinishedFunction failed.');
-                this.#reject(err);
-            }
-        }
-        finally {
-            this.#queryingIsFinished = false;
-        }
-    }
-    /**
-     * Cleans up resources.
-     */
-    async destroy() {
-        this.#resolve = null;
-        this.#reject = null;
-        betterClearInterval(this.#maybeRunInterval);
-    }
-}

package/autoscaling/concurrency_system.d.ts

@@ -1,268 +0,0 @@
-import type { CrawleeLogger } from '../log.js';
-import type { LoadSignalsOptions } from './snapshotter.js';
-import type { SystemInfo } from './system_status.js';
-export interface ConcurrencySystemOptions {
-    /**
-     * The minimum number of tasks running in parallel.
-     *
-     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
-     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
-     * @default 1
-     */
-    minConcurrency?: number;
-    /**
-     * The maximum number of tasks running in parallel.
-     * @default 200
-     */
-    maxConcurrency?: number;
-    /**
-     * The desired number of tasks that should be running parallel on the start of the pool,
-     * if there is a large enough supply of them.
-     * By default, it is `minConcurrency`.
-     */
-    desiredConcurrency?: number;
-    /**
-     * Minimum level of desired concurrency to reach before more scaling up is allowed.
-     * @default 0.90
-     */
-    desiredConcurrencyRatio?: number;
-    /**
-     * Defines the fractional amount of desired concurrency to be added with each scaling up.
-     * The minimum scaling step is one.
-     * @default 0.05
-     */
-    scaleUpStepRatio?: number;
-    /**
-     * Defines the amount of desired concurrency to be subtracted with each scaling down.
-     * The minimum scaling step is one.
-     * @default 0.05
-     */
-    scaleDownStepRatio?: number;
-    /**
-     * Specifies a period in which the instance logs its state, in seconds.
-     * Set to `null` to disable periodic logging.
-     * @default 60
-     */
-    loggingIntervalSecs?: number | null;
-    /**
-     * Defines in seconds how often the system should attempt to adjust the desired concurrency
-     * based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
-     * We suggest using a value from 5 to 20.
-     * @default 10
-     */
-    autoscaleIntervalSecs?: number;
-    /**
-     * The signals that tell the system whether the machine is overloaded: per-resource tuning for the built-in four
-     * (memory, event loop, CPU, storage backend) plus any {@link LoadSignalsOptions.custom|`custom`}
-     * implementations of your own. See {@link LoadSignalsOptions}.
-     */
-    loadSignals?: LoadSignalsOptions;
-    /**
-     * How far back the **autoscaling** decisions look, in seconds — the window the historical system status is
-     * evaluated over, and therefore how much history the signals retain (the memory cost of raising it).
-     * @default 30
-     */
-    snapshotHistorySecs?: number;
-    /**
-     * How far back the **task-gating** decision looks, in seconds — the window used to judge whether the system is
-     * overloaded *right now*, before dispatching one more task. Deliberately shorter than
-     * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`}, so that dispatch reacts to
-     * spikes quickly while scaling stays stable.
-     * @default 5
-     */
-    currentHistorySecs?: number;
-    /**
-     * The maximum number of tasks per minute the system can run.
-     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
-     */
-    maxTasksPerMinute?: number;
-    log?: CrawleeLogger;
-}
-/**
- * Identifies *who* is asking a governor for capacity: one {@link AutoscaledPool}, or the crawler driving it. The
- * same object is passed on every call a pool makes, so per-consumer state can be keyed off it or off its `id`.
- * @category Scaling
- */
-export interface ConcurrencyConsumer {
-    /** Process-unique and human-readable — a crawler's is its {@link BasicCrawlerOptions.id|`id`} option. */
-    readonly id: string;
-}
-/**
- * The contract between an {@link AutoscaledPool} and its concurrency "governor" — the object that answers *is
- * there free compute for one more task?* and tracks the budget that tasks are booked against.
- * {@link ConcurrencySystem} is the canonical implementation; the interface lets alternate governors be substituted
- * without depending on its internals.
- *
- * Every allocation method is told which {@link ConcurrencyConsumer|consumer} is asking, so an implementation can
- * allocate per consumer. {@link ConcurrencySystem} does not: it serves whoever asks first, which can starve a pool
- * that joins a saturated system late.
- * @category Scaling
- */
-export interface IConcurrencySystem {
-    /**
-     * The number of tasks that should currently be running in parallel, assuming a sufficient supply of them. How it
-     * is derived is up to the implementation, hence read-only here — but it must always be at least `1`, or a pool
-     * could never start the first task.
-     */
-    readonly desiredConcurrency: number;
-    /** The number of parallel tasks currently booked against this governor, regardless of which pool booked them. */
-    readonly currentConcurrency: number;
-    /**
-     * Whether the governor is ready to be booked against. {@link AutoscaledPool.run|`pool.run()`} refuses to run
-     * when this is `false`. An implementation with no startup lifecycle simply reports `true`.
-     */
-    readonly isRunning: boolean;
-    /**
-     * May **one more** task start right now, on behalf of `consumer`? A cheap pre-check the pool consults before
-     * querying task readiness.
-     *
-     * Must **not** enforce rate limits that only make sense for ready tasks (e.g. a per-minute task cap): the pool
-     * calls this before knowing whether any task is ready, so refusing here would stall an already-empty queue.
-     *
-     * Must also return `true` whenever `consumer` has nothing in flight of its own. A `false` sends that pool straight
-     * to its finished-check **without** consulting `isTaskReadyFunction`, so a governor that starves an idle pool can
-     * make its `run()` resolve while work is still pending. Tracking bookings per consumer answers that directly;
-     * {@link ConcurrencySystem}, which does not, instead never refuses while
-     * {@link IConcurrencySystem.currentConcurrency|`currentConcurrency`} is `0`.
-     */
-    hasCapacityForTask(consumer: ConcurrencyConsumer): boolean;
-    /**
-     * Books a task against the budget for `consumer`, returning `false` (without booking) when there is no room — the
-     * budget is spent, the consumer is over its share, or an implementation-specific rate limit was reached.
-     *
-     * Must be an *atomic* (synchronous) check-and-book: several pools may share one governor, and a check separated
-     * from the booking by an `await` lets two of them claim the last free slot at once.
-     */
-    tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
-    /** Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject). */
-    registerTaskEnd(consumer: ConcurrencyConsumer): void;
-}
-/**
- * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
- * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
- * over time.
- *
- * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
- * of letting each scale independently and oversubscribe the machine.
- *
- * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
- * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
- * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
- * system down for every borrower.
- * @category Scaling
- */
-export declare class ConcurrencySystem implements IConcurrencySystem {
-    #private;
-    private readonly log;
-    private readonly desiredConcurrencyRatio;
-    private readonly scaleUpStepRatio;
-    private readonly scaleDownStepRatio;
-    private readonly maxTasksPerMinute;
-    private _currentConcurrency;
-    private readonly snapshotter;
-    private readonly systemStatus;
-    constructor(options?: ConcurrencySystemOptions);
-    /**
-     * Gets the minimum number of tasks running in parallel.
-     */
-    get minConcurrency(): number;
-    /**
-     * Sets the minimum number of tasks running in parallel.
-     *
-     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
-     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
-     */
-    set minConcurrency(value: number);
-    /**
-     * Gets the maximum number of tasks running in parallel.
-     */
-    get maxConcurrency(): number;
-    /**
-     * Sets the maximum number of tasks running in parallel. Lowering it below the current
-     * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
-     * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
-     * limit as they settle).
-     */
-    set maxConcurrency(value: number);
-    /**
-     * Gets the desired concurrency for the system,
-     * which is an estimated number of parallel tasks that the system can currently support.
-     */
-    get desiredConcurrency(): number;
-    /**
-     * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
-     * in parallel if there's large enough supply of tasks.
-     */
-    set desiredConcurrency(value: number);
-    /**
-     * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
-     * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
-     * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
-     * that is the limit callers set in order to protect something.
-     */
-    private clampDesiredConcurrency;
-    get currentConcurrency(): number;
-    /** Whether the system is currently monitoring load and autoscaling the budget. */
-    get isRunning(): boolean;
-    /**
-     * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
-     * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
-     * signal fails to start.
-     */
-    start(): Promise<void>;
-    private boot;
-    /**
-     * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
-     */
-    stop(): Promise<void>;
-    private shutDown;
-    /**
-     * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
-     * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
-     * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
-     * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
-     */
-    private warnIfNotRunning;
-    /**
-     * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
-     * reached) or when the machine is overloaded past `minConcurrency`.
-     *
-     * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
-     * interface, letting the answer be queried directly.
-     */
-    hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
-    /** Whether the per-minute task cap has been reached. */
-    private get isOverMaxRequestLimit();
-    /**
-     * Atomically books a task against the shared budget: re-checks
-     * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
-     * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
-     * room. Call right before the task actually runs.
-     *
-     * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
-     * extra minute.
-     */
-    tryRegisterTaskStart(consumer?: ConcurrencyConsumer): boolean;
-    /** Returns a slot to the shared budget, whoever booked it. */
-    registerTaskEnd(_consumer?: ConcurrencyConsumer): void;
-    /**
-     * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
-     * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
-     * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
-     */
-    getCurrentStatus(): SystemInfo;
-    /**
-     * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
-     * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
-     */
-    private autoscale;
-    /**
-     * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
-     */
-    private scaleUp;
-    /**
-     * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
-     */
-    private scaleDown;
-    private incrementTasksDonePerSecond;
-}