@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/autoscaling/client_load_signal.js

@@ -1,73 +0,0 @@
-import { betterClearInterval, betterSetInterval } from '@apify/utilities';
-import { serviceLocator } from '../service_locator.js';
-import { SnapshotStore } from './load_signal.js';
-const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
-/**
- * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
- * exceeds a threshold.
- *
- * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
- *
- * Switch it off entirely ({@link LoadSignalsOptions.client|`client: false`}) if the storage backend reports no
- * rate-limit statistics, since it otherwise polls it every second to no purpose.
- *
- * @category Scaling
- */
-export class ClientLoadSignal {
-    name = 'clientInfo';
-    overloadedRatio;
-    store = new SnapshotStore();
-    intervalMillis;
-    maxErrors;
-    interval;
-    client;
-    constructor(options = {}) {
-        this.overloadedRatio = options.overloadedRatio ?? 0.3;
-        this.intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;
-        this.maxErrors = options.maxErrors ?? 3;
-        this.handle = this.handle.bind(this);
-    }
-    async start(context) {
-        this.store.useSampleWindow(context.maxSampleWindowMillis);
-        // A new session starts from a clean slate, or its first measurement diffs the error count against the previous
-        // session's — possibly against a different backend, since the client is resolved afresh just below.
-        this.store.clear();
-        // Resolved here rather than in the constructor, where asking for the backend would instantiate a default one
-        // as a side effect - long before the crawler that owns the run has had a chance to register its own.
-        this.client = serviceLocator.getStorageBackend();
-        this.interval = betterSetInterval(this.handle, this.intervalMillis);
-    }
-    async stop() {
-        if (this.interval)
-            betterClearInterval(this.interval);
-        this.interval = undefined;
-        this.client = undefined;
-    }
-    getSample(sampleDurationMillis) {
-        return this.store.getSample(sampleDurationMillis);
-    }
-    /**
-     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
-     * previous one.
-     * @internal Also lets tests drive the measurement without waiting on a timer.
-     */
-    handle(intervalCallback) {
-        const now = new Date();
-        const allErrorCounts = this.client?.stats?.rateLimitErrors ?? [];
-        const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
-        const snapshot = {
-            createdAt: now,
-            isOverloaded: false,
-            rateLimitErrorCount: currentErrCount,
-        };
-        const all = this.store.getAll();
-        const previousSnapshot = all[all.length - 1];
-        if (previousSnapshot) {
-            const delta = currentErrCount - previousSnapshot.rateLimitErrorCount;
-            if (delta > this.maxErrors)
-                snapshot.isOverloaded = true;
-        }
-        this.store.push(snapshot, now);
-        intervalCallback();
-    }
-}

package/autoscaling/concurrency_system.d.ts

@@ -1,283 +0,0 @@
-import type { CrawleeLogger } from '../log.js';
-import type { LoadSignalsOptions } from './snapshotter.js';
-import type { SystemInfo } from './system_status.js';
-export interface ConcurrencySystemOptions {
-    /**
-     * The minimum number of tasks running in parallel.
-     *
-     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
-     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
-     * @default 1
-     */
-    minConcurrency?: number;
-    /**
-     * The maximum number of tasks running in parallel.
-     * @default 200
-     */
-    maxConcurrency?: number;
-    /**
-     * The desired number of tasks that should be running parallel on the start of the pool,
-     * if there is a large enough supply of them.
-     * By default, it is `minConcurrency`.
-     */
-    desiredConcurrency?: number;
-    /**
-     * Minimum level of desired concurrency to reach before more scaling up is allowed.
-     * @default 0.90
-     */
-    desiredConcurrencyRatio?: number;
-    /**
-     * Defines the fractional amount of desired concurrency to be added with each scaling up.
-     * The minimum scaling step is one.
-     * @default 0.05
-     */
-    scaleUpStepRatio?: number;
-    /**
-     * Defines the amount of desired concurrency to be subtracted with each scaling down.
-     * The minimum scaling step is one.
-     * @default 0.05
-     */
-    scaleDownStepRatio?: number;
-    /**
-     * Specifies a period in which the instance logs its state, in seconds.
-     * Set to `null` to disable periodic logging.
-     * @default 60
-     */
-    loggingIntervalSecs?: number | null;
-    /**
-     * Defines in seconds how often the system should attempt to adjust the desired concurrency
-     * based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
-     * We suggest using a value from 5 to 20.
-     * @default 10
-     */
-    autoscaleIntervalSecs?: number;
-    /**
-     * The signals that tell the system whether the machine is overloaded: per-resource tuning for the built-in four
-     * (memory, event loop, CPU, client) plus any {@link LoadSignalsOptions.custom|`custom`} implementations of
-     * your own. See {@link LoadSignalsOptions}.
-     */
-    loadSignals?: LoadSignalsOptions;
-    /**
-     * How far back the **autoscaling** decisions look, in seconds — the window the historical system status is
-     * evaluated over, and therefore how much history the signals retain (the memory cost of raising it).
-     * @default 30
-     */
-    snapshotHistorySecs?: number;
-    /**
-     * How far back the **task-gating** decision looks, in seconds — the window used to judge whether the system is
-     * overloaded *right now*, before dispatching one more task. Deliberately shorter than
-     * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`}, so that dispatch reacts to
-     * spikes quickly while scaling stays stable.
-     * @default 5
-     */
-    currentHistorySecs?: number;
-    /**
-     * The maximum number of tasks per minute the system can run.
-     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
-     */
-    maxTasksPerMinute?: number;
-    log?: CrawleeLogger;
-}
-/**
- * Identifies *who* is asking a governor for capacity: one {@link AutoscaledPool}, or the crawler driving it. The
- * same object is passed on every call a pool makes, so per-consumer state can be keyed off it or off its `id`.
- * @category Scaling
- */
-export interface ConcurrencyConsumer {
-    /** Process-unique and human-readable — a crawler's is its {@link BasicCrawlerOptions.id|`id`} option. */
-    readonly id: string;
-}
-/**
- * The contract between an {@link AutoscaledPool} and its concurrency "governor" — the object that answers *is
- * there free compute for one more task?* and tracks the budget that tasks are booked against.
- * {@link ConcurrencySystem} is the canonical implementation; the interface lets alternate governors be substituted
- * without depending on its internals.
- *
- * Every allocation method is told which {@link ConcurrencyConsumer|consumer} is asking, so an implementation can
- * allocate per consumer. {@link ConcurrencySystem} does not: it serves whoever asks first, which can starve a pool
- * that joins a saturated system late.
- * @category Scaling
- */
-export interface IConcurrencySystem {
-    /**
-     * The number of tasks that should currently be running in parallel, assuming a sufficient supply of them. How it
-     * is derived is up to the implementation, hence read-only here — but it must always be at least `1`, or a pool
-     * could never start the first task.
-     */
-    readonly desiredConcurrency: number;
-    /** The number of parallel tasks currently booked against this governor, regardless of which pool booked them. */
-    readonly currentConcurrency: number;
-    /**
-     * Whether the governor is ready to be booked against. {@link AutoscaledPool.run|`pool.run()`} refuses to run
-     * when this is `false`. An implementation with no startup lifecycle simply reports `true`.
-     */
-    readonly isRunning: boolean;
-    /**
-     * May **one more** task start right now, on behalf of `consumer`? A cheap pre-check the pool consults before
-     * querying task readiness.
-     *
-     * Must **not** enforce rate limits that only make sense for ready tasks (e.g. a per-minute task cap): the pool
-     * calls this before knowing whether any task is ready, so refusing here would stall an already-empty queue.
-     *
-     * Must also return `true` whenever `consumer` has nothing in flight of its own. A `false` sends that pool straight
-     * to its finished-check **without** consulting `isTaskReadyFunction`, so a governor that starves an idle pool can
-     * make its `run()` resolve while work is still pending. Tracking bookings per consumer answers that directly;
-     * {@link ConcurrencySystem}, which does not, instead never refuses while
-     * {@link IConcurrencySystem.currentConcurrency|`currentConcurrency`} is `0`.
-     */
-    hasCapacityForTask(consumer: ConcurrencyConsumer): boolean;
-    /**
-     * Books a task against the budget for `consumer`, returning `false` (without booking) when there is no room — the
-     * budget is spent, the consumer is over its share, or an implementation-specific rate limit was reached.
-     *
-     * Must be an *atomic* (synchronous) check-and-book: several pools may share one governor, and a check separated
-     * from the booking by an `await` lets two of them claim the last free slot at once.
-     */
-    tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
-    /** Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject). */
-    registerTaskEnd(consumer: ConcurrencyConsumer): void;
-}
-/**
- * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
- * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
- * over time.
- *
- * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
- * of letting each scale independently and oversubscribe the machine.
- *
- * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
- * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
- * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
- * system down for every borrower.
- * @category Scaling
- */
-export declare class ConcurrencySystem implements IConcurrencySystem {
-    private readonly log;
-    private readonly desiredConcurrencyRatio;
-    private readonly scaleUpStepRatio;
-    private readonly scaleDownStepRatio;
-    private readonly loggingIntervalMillis;
-    private readonly autoscaleIntervalMillis;
-    private readonly maxTasksPerMinute;
-    private _minConcurrency;
-    private _maxConcurrency;
-    private _desiredConcurrency;
-    private _currentConcurrency;
-    private lastLoggingTime?;
-    private _tasksPerMinute;
-    private readonly snapshotter;
-    private readonly loadSignals;
-    private readonly systemStatus;
-    private autoscaleInterval?;
-    private tasksDonePerSecondInterval?;
-    /** Whether the snapshotter and autoscaling intervals are currently running. */
-    private running;
-    /** The in-flight (or completed) startup, memoized so concurrent `start()` calls await one boot. */
-    private startPromise?;
-    /** Set once per session, so a pool outliving `stop()` is reported once rather than every half second. */
-    private warnedAboutQueryWhileStopped;
-    constructor(options?: ConcurrencySystemOptions);
-    /**
-     * Gets the minimum number of tasks running in parallel.
-     */
-    get minConcurrency(): number;
-    /**
-     * Sets the minimum number of tasks running in parallel.
-     *
-     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
-     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
-     */
-    set minConcurrency(value: number);
-    /**
-     * Gets the maximum number of tasks running in parallel.
-     */
-    get maxConcurrency(): number;
-    /**
-     * Sets the maximum number of tasks running in parallel. Lowering it below the current
-     * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
-     * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
-     * limit as they settle).
-     */
-    set maxConcurrency(value: number);
-    /**
-     * Gets the desired concurrency for the system,
-     * which is an estimated number of parallel tasks that the system can currently support.
-     */
-    get desiredConcurrency(): number;
-    /**
-     * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
-     * in parallel if there's large enough supply of tasks.
-     */
-    set desiredConcurrency(value: number);
-    /**
-     * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
-     * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
-     * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
-     * that is the limit callers set in order to protect something.
-     */
-    private clampDesiredConcurrency;
-    get currentConcurrency(): number;
-    /** Whether the system is currently monitoring load and autoscaling the budget. */
-    get isRunning(): boolean;
-    /**
-     * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
-     * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
-     * signal fails to start.
-     */
-    start(): Promise<void>;
-    private boot;
-    /**
-     * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
-     */
-    stop(): Promise<void>;
-    private shutDown;
-    /**
-     * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
-     * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
-     * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
-     * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
-     */
-    private warnIfNotRunning;
-    /**
-     * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
-     * reached) or when the machine is overloaded past `minConcurrency`.
-     *
-     * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
-     * interface, letting the answer be queried directly.
-     */
-    hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
-    /** Whether the per-minute task cap has been reached. */
-    private get isOverMaxRequestLimit();
-    /**
-     * Atomically books a task against the shared budget: re-checks
-     * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
-     * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
-     * room. Call right before the task actually runs.
-     *
-     * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
-     * extra minute.
-     */
-    tryRegisterTaskStart(consumer?: ConcurrencyConsumer): boolean;
-    /** Returns a slot to the shared budget, whoever booked it. */
-    registerTaskEnd(_consumer?: ConcurrencyConsumer): void;
-    /**
-     * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
-     * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
-     * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
-     */
-    getCurrentStatus(): SystemInfo;
-    /**
-     * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
-     * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
-     */
-    private _autoscale;
-    /**
-     * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
-     */
-    private _scaleUp;
-    /**
-     * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
-     */
-    private _scaleDown;
-    private _incrementTasksDonePerSecond;
-}