@crawlee/http 4.0.0-beta.87 → 4.0.0-beta.89

package/internals/file-download.d.ts

@@ -68,7 +68,7 @@ export declare function ByteCounterStream({ logTransferredBytes, loggingInterval
  * ]
  * ```
  *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *

package/internals/file-download.js

@@ -87,7 +87,7 @@ export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000,
  * ]
  * ```
  *
- * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions` parameter of the `FileCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency` {@link AutoscaledPool} options are available directly in the `FileCrawler` constructor.
+ * New requests are only dispatched when there is enough free CPU and memory available, using the functionality provided by the {@link AutoscaledPool} class. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * ## Example usage
  *

package/internals/http-crawler.d.ts

@@ -1,9 +1,21 @@
-import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
+import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import type { JsonValue } from 'type-fest';
+/**
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
+ */
+export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
@@ -204,9 +216,9 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -283,6 +295,8 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
         maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -330,6 +344,13 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+    /**
+     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
+     * concurrency shortcuts on top. Not called for a supplied
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
+     * keep the tuning.
+     */
+    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
     private prepareHttpRequest;
     private makeHttpRequest;

package/internals/http-crawler.js

@@ -14,14 +14,25 @@ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpR
  */
 const HTML_AND_XML_MIME_TYPES = ['text/html', 'text/xml', 'application/xhtml+xml', 'application/xml'];
 const APPLICATION_JSON_MIME_TYPE = 'application/json';
-const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
+/**
+ * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
+ * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
+ *
+ * A {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} you supply yourself replaces that default
+ * wholesale, tuning included, so spread these options in if you want to keep it:
+ *
+ * ```typescript
+ * new ConcurrencySystem({ ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS, maxConcurrency: 50 });
+ * ```
+ */
+export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS = {
     desiredConcurrency: 10,
-    snapshotterOptions: {
-        eventLoopSnapshotIntervalSecs: 2,
-        maxBlockedMillis: 100,
-    },
-    systemStatusOptions: {
-        maxEventLoopOverloadedRatio: 0.7,
+    loadSignals: {
+        eventLoop: {
+            snapshotIntervalSecs: 2,
+            maxBlockedMillis: 100,
+            overloadedRatio: 0.7,
+        },
     },
 };
 /**
@@ -66,9 +77,9 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  *
  * New requests are only dispatched when there is enough free CPU and memory available,
  * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
- * parameter of the constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPool} options are available directly in the constructor.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * **Example usage:**
  *
@@ -124,10 +135,9 @@ export class HttpCrawler extends BasicCrawler {
         ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
         const { navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [], 
         // BasicCrawler
-        autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, contextPipelineBuilder, ...basicCrawlerOptions } = options;
+        contextPipelineBuilder, ...basicCrawlerOptions } = options;
         super({
             ...basicCrawlerOptions,
-            autoscaledPoolOptions,
             contextPipelineBuilder: contextPipelineBuilder ??
                 (() => this.buildContextPipeline()),
         });
@@ -151,6 +161,18 @@ export class HttpCrawler extends BasicCrawler {
         ];
         this.saveResponseCookies = saveResponseCookies;
     }
+    /**
+     * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
+     * concurrency shortcuts on top. Not called for a supplied
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} — spread the constant into it yourself to
+     * keep the tuning.
+     */
+    createDefaultConcurrencySystem(options) {
+        return super.createDefaultConcurrencySystem({
+            ...HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS,
+            ...options,
+        });
+    }
     buildContextPipeline() {
         // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
         // the response-derived members, so the guarded action is bypassed and the context left untouched.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.87",
+    "version": "4.0.0-beta.89",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.87",
-        "@crawlee/core": "4.0.0-beta.87",
-        "@crawlee/http-client": "4.0.0-beta.87",
-        "@crawlee/types": "4.0.0-beta.87",
-        "@crawlee/utils": "4.0.0-beta.87",
+        "@crawlee/basic": "4.0.0-beta.89",
+        "@crawlee/core": "4.0.0-beta.89",
+        "@crawlee/http-client": "4.0.0-beta.89",
+        "@crawlee/types": "4.0.0-beta.89",
+        "@crawlee/utils": "4.0.0-beta.89",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "1b7604cd77b694460aac4448b4555444fa1b2bdb"
+    "gitHead": "9279941162eb4be0a9113f768b5bd79d27e66eba"
 }