@crawlee/basic 4.0.0-beta.91 → 4.0.0-beta.93

package/internals/basic-crawler.d.ts

@@ -1,5 +1,5 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolPredicateOptions, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TypedRequestsLike } from '@crawlee/core';
-import { AutoscaledPool, ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
@@ -151,13 +151,14 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxCrawlDepth?: number;
     /**
-     * Lets you override the task-loop predicates (`isFinishedFunction`, `isTaskReadyFunction`) of the crawler's
-     * underlying {@link AutoscaledPool}. The `runTaskFunction` is owned by the crawler and cannot be overridden.
+     * Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
+     * start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
+     * through the pipeline — is owned by the crawler and cannot be overridden.
      *
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    autoscaledPoolOptions?: AutoscaledPoolPredicateOptions;
+    taskLoopOptions?: TaskLoopPredicates;
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
      * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -353,8 +354,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
  *
  * The crawler finishes if there are no more {@link Request} objects to crawl.
  *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
  * Concurrency is tuned via the {@link BasicCrawlerOptions.minConcurrency|`minConcurrency`},
  * {@link BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
  * {@link BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by
@@ -458,16 +459,26 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
     private concurrencySystemDep?;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that runs the crawler's task loop.
-     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     * The concurrency governor this run is booking its requests against — either the
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
+     * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * The pool only exposes read-only concurrency telemetry. To tune concurrency at runtime, keep a reference to a
-     * {@link ConcurrencySystem} and inject it via
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
+     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
+     * is also rebuilt for every run, so the instance is not stable across runs.
+     *
+     * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
+     * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
+     * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
+     */
+    get concurrencySystem(): IConcurrencySystem | undefined;
+    /**
+     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
+     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
+     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
+     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
+     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
-    autoscaledPool?: AutoscaledPool;
+    private autoscaledPool?;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -513,12 +524,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
     private ignoreHttpErrorStatusCodes;
     /**
-     * The resolved task-loop options for the crawler's own {@link AutoscaledPool} — the crawler-owned
-     * `runTaskFunction`, the (possibly user-overridden) ready/finished predicates and cadence/logging. Concurrency
-     * configuration lives on the {@link ConcurrencySystem} instead, and the pool's `consumer` identity is the
-     * crawler's own, so neither is settable here.
+     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
+     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
+     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
+     * settable here.
      */
-    private autoscaledPoolOptions;
+    private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
     private respectRobotsTxtFile;
@@ -554,7 +565,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -655,8 +666,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
     private getPeriodicLogger;
     /**
-     * Runs the crawler. Returns a promise that resolves once all the requests are processed
-     * and `autoscaledPool.isFinished` returns `true`.
+     * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
+     * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
+     * default "the request manager is empty") reports that the crawl is over.
      *
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
@@ -673,6 +685,20 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason?: string): void;
+    /**
+     * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
+     * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
+     * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
+     *
+     * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
+     * throughout, since a shared one may still be serving other crawlers.
+     */
+    pause(timeoutSecs?: number): Promise<void>;
+    /**
+     * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
+     * again. A no-op on a crawler that is not paused.
+     */
+    resume(): void;
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
      * if none has been configured or opened yet.

package/internals/basic-crawler.js

@@ -90,14 +90,26 @@ export class BasicCrawler {
     /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
     concurrencySystemDep;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that runs the crawler's task loop.
-     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     * The concurrency governor this run is booking its requests against — either the
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
+     * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * The pool only exposes read-only concurrency telemetry. To tune concurrency at runtime, keep a reference to a
-     * {@link ConcurrencySystem} and inject it via
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
+     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
+     * is also rebuilt for every run, so the instance is not stable across runs.
+     *
+     * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
+     * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
+     * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
+     */
+    get concurrencySystem() {
+        return this.concurrencySystemDep?.maybeValue;
+    }
+    /**
+     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
+     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
+     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
+     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
+     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
     autoscaledPool;
     /**
@@ -158,12 +170,12 @@ export class BasicCrawler {
     additionalHttpErrorStatusCodes;
     ignoreHttpErrorStatusCodes;
     /**
-     * The resolved task-loop options for the crawler's own {@link AutoscaledPool} — the crawler-owned
-     * `runTaskFunction`, the (possibly user-overridden) ready/finished predicates and cadence/logging. Concurrency
-     * configuration lives on the {@link ConcurrencySystem} instead, and the pool's `consumer` identity is the
-     * crawler's own, so neither is settable here.
+     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
+     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
+     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
+     * settable here.
      */
-    autoscaledPoolOptions;
+    taskLoopOptions;
     httpClient;
     retryOnBlocked;
     respectRobotsTxtFile;
@@ -189,7 +201,7 @@ export class BasicCrawler {
         sameDomainDelaySecs: ow.optional.number,
         maxRequestsPerCrawl: ow.optional.number,
         maxCrawlDepth: ow.optional.number,
-        autoscaledPoolOptions: ow.optional.object,
+        taskLoopOptions: ow.optional.object,
         concurrencySystem: ow.optional.object,
         sessionPool: ow.optional.object.validate(validators.sessionPool),
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
@@ -223,7 +235,7 @@ export class BasicCrawler {
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
-        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, autoscaledPoolOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
@@ -338,12 +350,12 @@ export class BasicCrawler {
             this.maxRequestsPerCrawl = maxRequestsPerCrawl;
             const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
             // eslint-disable-next-line prefer-const
-            let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
+            let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
             // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
             if (keepAlive) {
                 isFinishedFunction = async () => false;
             }
-            const basicCrawlerAutoscaledPoolConfiguration = {
+            const crawlerOwnedTaskLoopConfiguration = {
                 runTaskFunction: async () => {
                     const source = this.requestManager;
                     if (!source)
@@ -428,7 +440,7 @@ export class BasicCrawler {
                 },
                 log: this.log,
             };
-            this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
+            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
             this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
@@ -630,7 +642,7 @@ export class BasicCrawler {
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.autoscaledPool?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
             if (this.statusMessageCallback) {
                 await this.statusMessageCallback({
@@ -647,8 +659,9 @@ export class BasicCrawler {
         return { log, stop: () => clearInterval(interval) };
     }
     /**
-     * Runs the crawler. Returns a promise that resolves once all the requests are processed
-     * and `autoscaledPool.isFinished` returns `true`.
+     * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
+     * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
+     * default "the request manager is empty") reports that the crawl is over.
      *
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
@@ -773,6 +786,32 @@ export class BasicCrawler {
         this.log.info(reason);
         this.unexpectedStop = true;
     }
+    /**
+     * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
+     * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
+     * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
+     *
+     * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
+     * throughout, since a shared one may still be serving other crawlers.
+     */
+    async pause(timeoutSecs) {
+        if (!this.autoscaledPool) {
+            this.log.warning('Cannot pause a crawler that is not running.');
+            return;
+        }
+        await this.autoscaledPool.pause(timeoutSecs);
+    }
+    /**
+     * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
+     * again. A no-op on a crawler that is not paused.
+     */
+    resume() {
+        if (!this.autoscaledPool) {
+            this.log.warning('Cannot resume a crawler that is not running.');
+            return;
+        }
+        this.autoscaledPool.resume();
+    }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
      * if none has been configured or opened yet.
@@ -1035,7 +1074,7 @@ export class BasicCrawler {
         this.concurrencySystemDep = this.resolveConcurrencySystem();
         await this.concurrencySystemDep.ifOwned((system) => system.start());
         this.autoscaledPool = new AutoscaledPool({
-            ...this.autoscaledPoolOptions,
+            ...this.taskLoopOptions,
             concurrencySystem: this.concurrencySystemDep.value,
             consumer: this.identity,
         });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.91",
+    "version": "4.0.0-beta.93",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.91",
-        "@crawlee/http-client": "4.0.0-beta.91",
-        "@crawlee/types": "4.0.0-beta.91",
-        "@crawlee/utils": "4.0.0-beta.91",
+        "@crawlee/core": "4.0.0-beta.93",
+        "@crawlee/http-client": "4.0.0-beta.93",
+        "@crawlee/types": "4.0.0-beta.93",
+        "@crawlee/utils": "4.0.0-beta.93",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.91"
+        "@crawlee/impit-client": "^4.0.0-beta.93"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "1e3e1ca10f24be53e1d527b3ea25f456baebecbf"
+    "gitHead": "b9d21e80d94d01e21d4e1c19191610d0157dd172"
 }