@crawlee/basic 4.0.0-beta.88 → 4.0.0-beta.89

package/internals/basic-crawler.d.ts

@@ -1,5 +1,5 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TypedRequestsLike } from '@crawlee/core';
-import { AutoscaledPool, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolPredicateOptions, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TypedRequestsLike } from '@crawlee/core';
+import { AutoscaledPool, ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
@@ -151,29 +151,49 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxCrawlDepth?: number;
     /**
-     * Custom options passed to the underlying {@link AutoscaledPool} constructor.
-     * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
-     * option is provided by the crawler and cannot be overridden.
-     * However, we can provide custom implementations of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}
-     * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`}.
+     * Lets you override the task-loop predicates (`isFinishedFunction`, `isTaskReadyFunction`) of the crawler's
+     * underlying {@link AutoscaledPool}. The `runTaskFunction` is owned by the crawler and cannot be overridden.
+     *
+     * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
+     */
+    autoscaledPoolOptions?: AutoscaledPoolPredicateOptions;
+    /**
+     * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
+     * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
+     * scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
+     * lives on the instance itself.
+     *
+     * Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
+     * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
+     * accounting is shared.
+     *
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
+     * the default system this one replaces — combining the two throws.
+     *
+     * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
+     * every crawler borrowing it has finished. The crawler does neither on your behalf.
      */
-    autoscaledPoolOptions?: AutoscaledPoolOptions;
+    concurrencySystem?: IConcurrencySystem;
     /**
      * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
+     * {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}.
      * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
      * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
      */
     minConcurrency?: number;
     /**
      * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
+     * {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
-     * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
+     * Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
+     * crawler's default {@link ConcurrencySystem}.
      */
     maxRequestsPerMinute?: number;
     /**
@@ -335,11 +355,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
+ * Concurrency is tuned via the {@link BasicCrawlerOptions.minConcurrency|`minConcurrency`},
+ * {@link BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
+ * {@link BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by
+ * injecting a pre-configured {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -431,11 +450,22 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private requestManagerTimeoutsApplied;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
+     * Resolves the governor for one run: either the injected
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
+     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
+     */
+    private readonly resolveConcurrencySystem;
+    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    private concurrencySystemDep?;
+    /**
+     * A reference to the underlying {@link AutoscaledPool} class that runs the crawler's task loop.
      * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to change the concurrency settings on the fly,
-     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
+     * We can use it to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
      * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     *
+     * The pool only exposes read-only concurrency telemetry. To tune concurrency at runtime, keep a reference to a
+     * {@link ConcurrencySystem} and inject it via
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
      */
     autoscaledPool?: AutoscaledPool;
     /**
@@ -482,6 +512,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
     private ignoreHttpErrorStatusCodes;
+    /**
+     * The resolved task-loop options for the crawler's own {@link AutoscaledPool} — the crawler-owned
+     * `runTaskFunction`, the (possibly user-overridden) ready/finished predicates and cadence/logging. Concurrency
+     * configuration lives on the {@link ConcurrencySystem} instead, and the pool's `consumer` identity is the
+     * crawler's own, so neither is settable here.
+     */
     private autoscaledPoolOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
@@ -519,6 +555,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -566,6 +604,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
+    /**
+     * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
+     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
+     *
+     * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
+     * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
+     */
+    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
     /**
      * Determines if the given HTTP status code is an error status code given
      * the default behaviour and user-set preferences.

package/internals/basic-crawler.js

@@ -1,11 +1,11 @@
 import { writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
+import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
 import { FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
 import { ensureDir, writeJSON } from 'fs-extra/esm';
-import ow from 'ow';
+import ow, { ArgumentError } from 'ow';
 import { getDomain } from 'tldts';
 import { LruCache } from '@apify/datastructures';
 import { addTimeoutToPromise, TimeoutError } from '@apify/timeout';
@@ -82,11 +82,22 @@ export class BasicCrawler {
      */
     requestManagerTimeoutsApplied = false;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
+     * Resolves the governor for one run: either the injected
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
+     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
+     */
+    resolveConcurrencySystem;
+    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    concurrencySystemDep;
+    /**
+     * A reference to the underlying {@link AutoscaledPool} class that runs the crawler's task loop.
      * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to change the concurrency settings on the fly,
-     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
+     * We can use it to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
      * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     *
+     * The pool only exposes read-only concurrency telemetry. To tune concurrency at runtime, keep a reference to a
+     * {@link ConcurrencySystem} and inject it via
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
      */
     autoscaledPool;
     /**
@@ -146,6 +157,12 @@ export class BasicCrawler {
     blockedStatusCodes = new Set();
     additionalHttpErrorStatusCodes;
     ignoreHttpErrorStatusCodes;
+    /**
+     * The resolved task-loop options for the crawler's own {@link AutoscaledPool} — the crawler-owned
+     * `runTaskFunction`, the (possibly user-overridden) ready/finished predicates and cadence/logging. Concurrency
+     * configuration lives on the {@link ConcurrencySystem} instead, and the pool's `consumer` identity is the
+     * crawler's own, so neither is settable here.
+     */
     autoscaledPoolOptions;
     httpClient;
     retryOnBlocked;
@@ -173,6 +190,7 @@ export class BasicCrawler {
         maxRequestsPerCrawl: ow.optional.number,
         maxCrawlDepth: ow.optional.number,
         autoscaledPoolOptions: ow.optional.object,
+        concurrencySystem: ow.optional.object,
         sessionPool: ow.optional.object.validate(validators.sessionPool),
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
         statusMessageLoggingInterval: ow.optional.number,
@@ -205,11 +223,20 @@ export class BasicCrawler {
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
-        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, autoscaledPoolOptions = {}, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, autoscaledPoolOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
         minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, id, } = options;
+        // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
+        // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
+        // hammering a site.
+        if (concurrencySystem !== undefined &&
+            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
+            throw new ArgumentError('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
+                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
+                'replaces. Pass them to the `ConcurrencySystem` constructor instead.', this.constructor);
+        }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
         // (via AsyncLocalStorage) resolve to this scoped instance instead.
@@ -317,9 +344,6 @@ export class BasicCrawler {
                 isFinishedFunction = async () => false;
             }
             const basicCrawlerAutoscaledPoolConfiguration = {
-                minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,
-                maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,
-                maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,
                 runTaskFunction: async () => {
                     const source = this.requestManager;
                     if (!source)
@@ -405,11 +429,28 @@ export class BasicCrawler {
                 log: this.log,
             };
             this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
+            this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+                minConcurrency,
+                maxConcurrency,
+                maxTasksPerMinute: maxRequestsPerMinute,
+                log: this.log,
+            }));
         }
         finally {
             serviceLocatorScope.exitScope();
         }
     }
+    /**
+     * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
+     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
+     *
+     * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
+     * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
+     */
+    createDefaultConcurrencySystem(options) {
+        return new ConcurrencySystem(options);
+    }
     /**
      * Determines if the given HTTP status code is an error status code given
      * the default behaviour and user-set preferences.
@@ -648,8 +689,19 @@ export class BasicCrawler {
         if (requests) {
             await this.addRequests(requests, addRequestsOptions);
         }
-        await this._init();
-        await this.stats.startCapturing();
+        try {
+            await this._init();
+            await this.stats.startCapturing();
+        }
+        catch (error) {
+            // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
+            await this.teardown().catch((teardownError) => {
+                this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+            });
+            // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
+            this.running = false;
+            throw error;
+        }
         const periodicLogger = this.getPeriodicLogger();
         this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
         const sigintHandler = async () => {
@@ -977,7 +1029,16 @@ export class BasicCrawler {
             await eventManager.init();
             this._closeEvents = true;
         }
-        this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
+        // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
+        // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
+        // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
+        this.concurrencySystemDep = this.resolveConcurrencySystem();
+        await this.concurrencySystemDep.ifOwned((system) => system.start());
+        this.autoscaledPool = new AutoscaledPool({
+            ...this.autoscaledPoolOptions,
+            concurrencySystem: this.concurrencySystemDep.value,
+            consumer: this.identity,
+        });
         await this.getRequestManager();
     }
     async runRequestHandler(crawlingContext) {
@@ -1356,6 +1417,7 @@ export class BasicCrawler {
         }
         await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
         await this.autoscaledPool?.abort();
+        await this.concurrencySystemDep?.ifOwned((system) => system.stop());
     }
     _getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.88",
+    "version": "4.0.0-beta.89",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.88",
-        "@crawlee/http-client": "4.0.0-beta.88",
-        "@crawlee/types": "4.0.0-beta.88",
-        "@crawlee/utils": "4.0.0-beta.88",
+        "@crawlee/core": "4.0.0-beta.89",
+        "@crawlee/http-client": "4.0.0-beta.89",
+        "@crawlee/types": "4.0.0-beta.89",
+        "@crawlee/utils": "4.0.0-beta.89",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.88"
+        "@crawlee/impit-client": "^4.0.0-beta.89"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "4f7d17669d59250cf802591e73498793de28967d"
+    "gitHead": "9279941162eb4be0a9113f768b5bd79d27e66eba"
 }