@crawlee/basic 4.0.0-beta.144 → 4.0.0-beta.145

package/internals/basic-crawler.d.ts

@@ -176,8 +176,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
      * accounting is shared.
      *
-     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
-     * the default system this one replaces — combining the two throws.
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
      *
      * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
      * every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -197,6 +197,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -478,6 +484,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -489,7 +496,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting

package/internals/basic-crawler.js

@@ -271,6 +271,7 @@ export class BasicCrawler {
         // AutoscaledPool shorthands
         minConcurrency: schemas.anyNumber.optional(),
         maxConcurrency: schemas.anyNumber.optional(),
+        initialConcurrency: schemas.anyNumber.optional(),
         maxRequestsPerMinute: schemas.anyNumber
             .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
             .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -293,15 +294,18 @@ export class BasicCrawler {
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
+        minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
         if (concurrencySystem !== undefined &&
-            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
-            throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
-                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
+            (minConcurrency !== undefined ||
+                maxConcurrency !== undefined ||
+                initialConcurrency !== undefined ||
+                maxRequestsPerMinute !== undefined)) {
+            throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+                'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+                'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
         }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -528,6 +532,9 @@ export class BasicCrawler {
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
+                // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+                // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+                ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
                 log: this.log,
             }));
         }
@@ -537,7 +544,7 @@ export class BasicCrawler {
     }
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.144",
+    "version": "4.0.0-beta.145",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.144",
-        "@crawlee/http-client": "4.0.0-beta.144",
-        "@crawlee/types": "4.0.0-beta.144",
-        "@crawlee/utils": "4.0.0-beta.144",
+        "@crawlee/core": "4.0.0-beta.145",
+        "@crawlee/http-client": "4.0.0-beta.145",
+        "@crawlee/types": "4.0.0-beta.145",
+        "@crawlee/utils": "4.0.0-beta.145",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.144"
+        "@crawlee/impit-client": "^4.0.0-beta.145"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "4acf68a225909523047f872cbdeb87428bfc560d"
+    "gitHead": "e8b4d50265b9e433ae61c69d16cc1fee00a4d7e9"
 }