@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/autoscaling/client_load_signal.d.ts

@@ -0,0 +1,55 @@
+import type { LoadSignal, LoadSignalStartContext, LoadSnapshot } from './load_signal.js';
+/**
+ * A snapshot produced by the built-in client (rate-limit) signal.
+ * @internal
+ */
+export interface ClientSnapshot extends LoadSnapshot {
+    rateLimitErrorCount: number;
+}
+/**
+ * Tuning for the built-in **client** (rate-limit) load signal, as accepted both by {@link ClientLoadSignal} and by
+ * the {@link LoadSignalsOptions.client|`client`} shorthand on {@link LoadSignalsOptions}.
+ */
+export interface ClientLoadSignalOptions {
+    /**
+     * Defines the interval of checking the current state of the remote API client, in seconds.
+     * @default 1
+     */
+    snapshotIntervalSecs?: number;
+    /**
+     * Defines the maximum number of new rate limit errors within the given interval.
+     * @default 3
+     */
+    maxErrors?: number;
+    /**
+     * Maximum ratio of overloaded snapshots in a sample before the client counts as overloaded.
+     * @default 0.3
+     */
+    overloadedRatio?: number;
+}
+/**
+ * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
+ * exceeds a threshold.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * Switch it off entirely ({@link LoadSignalsOptions.client|`client: false`}) if the storage backend reports no
+ * rate-limit statistics, since it otherwise polls it every second to no purpose.
+ *
+ * @category Scaling
+ */
+export declare class ClientLoadSignal implements LoadSignal {
+    #private;
+    readonly name = "clientInfo";
+    readonly overloadedRatio: number;
+    constructor(options?: ClientLoadSignalOptions);
+    start(context: LoadSignalStartContext): Promise<void>;
+    stop(): Promise<void>;
+    getSample(sampleDurationMillis?: number): LoadSnapshot[];
+    /**
+     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
+     * previous one.
+     * @internal Also lets tests drive the measurement without waiting on a timer.
+     */
+    handle(intervalCallback: () => unknown): void;
+}

package/autoscaling/client_load_signal.js

@@ -0,0 +1,73 @@
+import { betterClearInterval, betterSetInterval } from '@apify/utilities';
+import { serviceLocator } from '../service_locator.js';
+import { SnapshotStore } from './load_signal.js';
+const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
+/**
+ * Periodically checks the storage backend for rate-limit errors (HTTP 429) and reports overload when the error delta
+ * exceeds a threshold.
+ *
+ * Built by default; construct one yourself only to wrap or adapt it — see {@link LoadSignal}.
+ *
+ * Switch it off entirely ({@link LoadSignalsOptions.client|`client: false`}) if the storage backend reports no
+ * rate-limit statistics, since it otherwise polls it every second to no purpose.
+ *
+ * @category Scaling
+ */
+export class ClientLoadSignal {
+    name = 'clientInfo';
+    overloadedRatio;
+    #store = new SnapshotStore();
+    #intervalMillis;
+    #maxErrors;
+    #interval;
+    #client;
+    constructor(options = {}) {
+        this.overloadedRatio = options.overloadedRatio ?? 0.3;
+        this.#intervalMillis = (options.snapshotIntervalSecs ?? 1) * 1000;
+        this.#maxErrors = options.maxErrors ?? 3;
+        this.handle = this.handle.bind(this);
+    }
+    async start(context) {
+        this.#store.useSampleWindow(context.maxSampleWindowMillis);
+        // A new session starts from a clean slate, or its first measurement diffs the error count against the previous
+        // session's — possibly against a different backend, since the client is resolved afresh just below.
+        this.#store.clear();
+        // Resolved here rather than in the constructor, where asking for the backend would instantiate a default one
+        // as a side effect - long before the crawler that owns the run has had a chance to register its own.
+        this.#client = serviceLocator.getStorageBackend();
+        this.#interval = betterSetInterval(this.handle, this.#intervalMillis);
+    }
+    async stop() {
+        if (this.#interval)
+            betterClearInterval(this.#interval);
+        this.#interval = undefined;
+        this.#client = undefined;
+    }
+    getSample(sampleDurationMillis) {
+        return this.#store.getSample(sampleDurationMillis);
+    }
+    /**
+     * Records one snapshot, overloaded when rate-limit errors grew by more than the configured limit since the
+     * previous one.
+     * @internal Also lets tests drive the measurement without waiting on a timer.
+     */
+    handle(intervalCallback) {
+        const now = new Date();
+        const allErrorCounts = this.#client?.stats?.rateLimitErrors ?? [];
+        const currentErrCount = allErrorCounts[CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT] || 0;
+        const snapshot = {
+            createdAt: now,
+            isOverloaded: false,
+            rateLimitErrorCount: currentErrCount,
+        };
+        const all = this.#store.getAll();
+        const previousSnapshot = all[all.length - 1];
+        if (previousSnapshot) {
+            const delta = currentErrCount - previousSnapshot.rateLimitErrorCount;
+            if (delta > this.#maxErrors)
+                snapshot.isOverloaded = true;
+        }
+        this.#store.push(snapshot, now);
+        intervalCallback();
+    }
+}

package/autoscaling/concurrency_system.d.ts

@@ -0,0 +1,268 @@
+import type { CrawleeLogger } from '../log.js';
+import type { LoadSignalsOptions } from './snapshotter.js';
+import type { SystemInfo } from './system_status.js';
+export interface ConcurrencySystemOptions {
+    /**
+     * The minimum number of tasks running in parallel.
+     *
+     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
+     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
+     * @default 1
+     */
+    minConcurrency?: number;
+    /**
+     * The maximum number of tasks running in parallel.
+     * @default 200
+     */
+    maxConcurrency?: number;
+    /**
+     * The desired number of tasks that should be running parallel on the start of the pool,
+     * if there is a large enough supply of them.
+     * By default, it is `minConcurrency`.
+     */
+    desiredConcurrency?: number;
+    /**
+     * Minimum level of desired concurrency to reach before more scaling up is allowed.
+     * @default 0.90
+     */
+    desiredConcurrencyRatio?: number;
+    /**
+     * Defines the fractional amount of desired concurrency to be added with each scaling up.
+     * The minimum scaling step is one.
+     * @default 0.05
+     */
+    scaleUpStepRatio?: number;
+    /**
+     * Defines the amount of desired concurrency to be subtracted with each scaling down.
+     * The minimum scaling step is one.
+     * @default 0.05
+     */
+    scaleDownStepRatio?: number;
+    /**
+     * Specifies a period in which the instance logs its state, in seconds.
+     * Set to `null` to disable periodic logging.
+     * @default 60
+     */
+    loggingIntervalSecs?: number | null;
+    /**
+     * Defines in seconds how often the system should attempt to adjust the desired concurrency
+     * based on the latest system status. Setting it lower than 1 might have a severe impact on performance.
+     * We suggest using a value from 5 to 20.
+     * @default 10
+     */
+    autoscaleIntervalSecs?: number;
+    /**
+     * The signals that tell the system whether the machine is overloaded: per-resource tuning for the built-in four
+     * (memory, event loop, CPU, client) plus any {@link LoadSignalsOptions.custom|`custom`} implementations of
+     * your own. See {@link LoadSignalsOptions}.
+     */
+    loadSignals?: LoadSignalsOptions;
+    /**
+     * How far back the **autoscaling** decisions look, in seconds — the window the historical system status is
+     * evaluated over, and therefore how much history the signals retain (the memory cost of raising it).
+     * @default 30
+     */
+    snapshotHistorySecs?: number;
+    /**
+     * How far back the **task-gating** decision looks, in seconds — the window used to judge whether the system is
+     * overloaded *right now*, before dispatching one more task. Deliberately shorter than
+     * {@link ConcurrencySystemOptions.snapshotHistorySecs|`snapshotHistorySecs`}, so that dispatch reacts to
+     * spikes quickly while scaling stays stable.
+     * @default 5
+     */
+    currentHistorySecs?: number;
+    /**
+     * The maximum number of tasks per minute the system can run.
+     * By default, this is set to `Infinity`, but you can pass any positive, non-zero integer.
+     */
+    maxTasksPerMinute?: number;
+    log?: CrawleeLogger;
+}
+/**
+ * Identifies *who* is asking a governor for capacity: one {@link AutoscaledPool}, or the crawler driving it. The
+ * same object is passed on every call a pool makes, so per-consumer state can be keyed off it or off its `id`.
+ * @category Scaling
+ */
+export interface ConcurrencyConsumer {
+    /** Process-unique and human-readable — a crawler's is its {@link BasicCrawlerOptions.id|`id`} option. */
+    readonly id: string;
+}
+/**
+ * The contract between an {@link AutoscaledPool} and its concurrency "governor" — the object that answers *is
+ * there free compute for one more task?* and tracks the budget that tasks are booked against.
+ * {@link ConcurrencySystem} is the canonical implementation; the interface lets alternate governors be substituted
+ * without depending on its internals.
+ *
+ * Every allocation method is told which {@link ConcurrencyConsumer|consumer} is asking, so an implementation can
+ * allocate per consumer. {@link ConcurrencySystem} does not: it serves whoever asks first, which can starve a pool
+ * that joins a saturated system late.
+ * @category Scaling
+ */
+export interface IConcurrencySystem {
+    /**
+     * The number of tasks that should currently be running in parallel, assuming a sufficient supply of them. How it
+     * is derived is up to the implementation, hence read-only here — but it must always be at least `1`, or a pool
+     * could never start the first task.
+     */
+    readonly desiredConcurrency: number;
+    /** The number of parallel tasks currently booked against this governor, regardless of which pool booked them. */
+    readonly currentConcurrency: number;
+    /**
+     * Whether the governor is ready to be booked against. {@link AutoscaledPool.run|`pool.run()`} refuses to run
+     * when this is `false`. An implementation with no startup lifecycle simply reports `true`.
+     */
+    readonly isRunning: boolean;
+    /**
+     * May **one more** task start right now, on behalf of `consumer`? A cheap pre-check the pool consults before
+     * querying task readiness.
+     *
+     * Must **not** enforce rate limits that only make sense for ready tasks (e.g. a per-minute task cap): the pool
+     * calls this before knowing whether any task is ready, so refusing here would stall an already-empty queue.
+     *
+     * Must also return `true` whenever `consumer` has nothing in flight of its own. A `false` sends that pool straight
+     * to its finished-check **without** consulting `isTaskReadyFunction`, so a governor that starves an idle pool can
+     * make its `run()` resolve while work is still pending. Tracking bookings per consumer answers that directly;
+     * {@link ConcurrencySystem}, which does not, instead never refuses while
+     * {@link IConcurrencySystem.currentConcurrency|`currentConcurrency`} is `0`.
+     */
+    hasCapacityForTask(consumer: ConcurrencyConsumer): boolean;
+    /**
+     * Books a task against the budget for `consumer`, returning `false` (without booking) when there is no room — the
+     * budget is spent, the consumer is over its share, or an implementation-specific rate limit was reached.
+     *
+     * Must be an *atomic* (synchronous) check-and-book: several pools may share one governor, and a check separated
+     * from the booking by an `await` lets two of them claim the last free slot at once.
+     */
+    tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
+    /** Returns a task's slot to `consumer`'s budget. Called once the task settles (resolve or reject). */
+    registerTaskEnd(consumer: ConcurrencyConsumer): void;
+}
+/**
+ * The shareable "governor" behind an {@link AutoscaledPool}: it decides whether there is free compute for one more
+ * task by combining live system load (via an internal {@link Snapshotter}) with a concurrency budget it autoscales
+ * over time.
+ *
+ * Sharing one instance between several pools (and therefore several crawlers) caps their *combined* compute, instead
+ * of letting each scale independently and oversubscribe the machine.
+ *
+ * Whoever builds the instance owns its lifecycle: call {@link ConcurrencySystem.start|`start()`} before any
+ * borrowing pool runs and {@link ConcurrencySystem.stop|`stop()`} once they are all done (crawlers do this for the
+ * default system they build, never for an injected one). Both calls are idempotent, and the first `stop()` tears the
+ * system down for every borrower.
+ * @category Scaling
+ */
+export declare class ConcurrencySystem implements IConcurrencySystem {
+    #private;
+    private readonly log;
+    private readonly desiredConcurrencyRatio;
+    private readonly scaleUpStepRatio;
+    private readonly scaleDownStepRatio;
+    private readonly maxTasksPerMinute;
+    private _currentConcurrency;
+    private readonly snapshotter;
+    private readonly systemStatus;
+    constructor(options?: ConcurrencySystemOptions);
+    /**
+     * Gets the minimum number of tasks running in parallel.
+     */
+    get minConcurrency(): number;
+    /**
+     * Sets the minimum number of tasks running in parallel.
+     *
+     * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your code might run extremely slow or crash.
+     * If you're not sure, just keep the default value and the concurrency will scale up automatically.
+     */
+    set minConcurrency(value: number);
+    /**
+     * Gets the maximum number of tasks running in parallel.
+     */
+    get maxConcurrency(): number;
+    /**
+     * Sets the maximum number of tasks running in parallel. Lowering it below the current
+     * {@link ConcurrencySystem.desiredConcurrency|`desiredConcurrency`} pulls that down to the new ceiling too, so
+     * the change takes effect immediately (in-flight tasks are never cancelled — the budget simply drains to the new
+     * limit as they settle).
+     */
+    set maxConcurrency(value: number);
+    /**
+     * Gets the desired concurrency for the system,
+     * which is an estimated number of parallel tasks that the system can currently support.
+     */
+    get desiredConcurrency(): number;
+    /**
+     * Sets the desired concurrency for the system, i.e. the number of tasks that should be running
+     * in parallel if there's large enough supply of tasks.
+     */
+    set desiredConcurrency(value: number);
+    /**
+     * Re-establishes `minConcurrency <= desiredConcurrency <= maxConcurrency` after any of the three is retuned.
+     * Dispatch gates on the desired value alone, so one stranded above `maxConcurrency` would make the ceiling
+     * meaningless. A contradictory pair (`minConcurrency > maxConcurrency`) resolves in favour of the maximum, since
+     * that is the limit callers set in order to protect something.
+     */
+    private clampDesiredConcurrency;
+    get currentConcurrency(): number;
+    /** Whether the system is currently monitoring load and autoscaling the budget. */
+    get isRunning(): boolean;
+    /**
+     * Boots the underlying snapshotter and the autoscaling interval. Idempotent, so a shared system isn't restarted
+     * when handed to another consumer; concurrent callers await one startup. Rejects, leaving nothing running, if a
+     * signal fails to start.
+     */
+    start(): Promise<void>;
+    private boot;
+    /**
+     * Stops the snapshotter and intervals. Idempotent and safe to call even if the system was never started.
+     */
+    stop(): Promise<void>;
+    private shutDown;
+    /**
+     * Reports, once per session, that capacity is being queried on a system that isn't running — a mistake nothing
+     * else catches, since {@link AutoscaledPool.run|`pool.run()`} only checks
+     * {@link ConcurrencySystem.isRunning|`isRunning`} on the way in. Both the overload verdict and
+     * `desiredConcurrency` are frozen at that point, so the borrowing pool would otherwise just quietly mis-scale.
+     */
+    private warnIfNotRunning;
+    /**
+     * May **one more** task start right now? Returns `false` when the shared budget is spent (desired concurrency
+     * reached) or when the machine is overloaded past `minConcurrency`.
+     *
+     * One budget for the whole machine, so the asking consumer is ignored — and therefore optional here, unlike in the
+     * interface, letting the answer be queried directly.
+     */
+    hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
+    /** Whether the per-minute task cap has been reached. */
+    private get isOverMaxRequestLimit();
+    /**
+     * Atomically books a task against the shared budget: re-checks
+     * {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} plus the per-minute task cap and
+     * increments the current concurrency in one synchronous step, returning `false` (without booking) when there is no
+     * room. Call right before the task actually runs.
+     *
+     * The cap is enforced here rather than in the pre-check so that an empty queue never blocks the pool for a whole
+     * extra minute.
+     */
+    tryRegisterTaskStart(consumer?: ConcurrencyConsumer): boolean;
+    /** Returns a slot to the shared budget, whoever booked it. */
+    registerTaskEnd(_consumer?: ConcurrencyConsumer): void;
+    /**
+     * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
+     * task-gating window, exactly as {@link ConcurrencySystem.hasCapacityForTask|`hasCapacityForTask()`} sees them.
+     * The one public window into load monitoring — useful for answering *why* a crawl is not scaling up.
+     */
+    getCurrentStatus(): SystemInfo;
+    /**
+     * Evaluates the historical system status and scales the shared desired concurrency up or down accordingly. Driven
+     * by the autoscaling interval started in {@link ConcurrencySystem.start|`start()`}.
+     */
+    private autoscale;
+    /**
+     * Scales the system up by increasing the desired concurrency by the scaleUpStepRatio.
+     */
+    private scaleUp;
+    /**
+     * Scales the system down by decreasing the desired concurrency by the scaleDownStepRatio.
+     */
+    private scaleDown;
+    private incrementTasksDonePerSecond;
+}