@crawlee/basic 4.0.0-beta.143 → 4.0.0-beta.145

package/internals/basic-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopOptions, TypedRequestsLike } from '@crawlee/core';
 import { ConcurrencySystem, Configuration, ContextPipeline, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
 import { BaseHttpClient } from '@crawlee/http-client';
 import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
@@ -165,7 +165,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
      * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -176,8 +176,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
      * accounting is shared.
      *
-     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
-     * the default system this one replaces — combining the two throws.
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
      *
      * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
      * every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -197,6 +197,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -430,18 +436,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     hasFinishedBefore: boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
-    private requestHandlerTimeoutMillis;
     protected readonly internalTimeoutMillis: number;
     private get handledRequestsCount();
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    /**
-     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
-     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
-     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
-     * settable here.
-     */
-    private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
     /**
@@ -486,6 +484,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -497,7 +496,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting

package/internals/basic-crawler.js

@@ -7,7 +7,7 @@ import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
@@ -193,8 +193,7 @@ export class BasicCrawler {
     requestHandler;
     #errorHandler;
     #failedRequestHandler;
-    // kept as TS-private: tests read it at runtime
-    requestHandlerTimeoutMillis;
+    #requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     #maxRequestRetries;
     #maxCrawlDepth;
@@ -214,8 +213,7 @@ export class BasicCrawler {
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
-    // kept as TS-private: tests mutate it at runtime
-    taskLoopOptions;
+    #taskLoopOptions;
     httpClient;
     retryOnBlocked;
     #respectRobotsTxtFile;
@@ -273,6 +271,7 @@ export class BasicCrawler {
         // AutoscaledPool shorthands
         minConcurrency: schemas.anyNumber.optional(),
         maxConcurrency: schemas.anyNumber.optional(),
+        initialConcurrency: schemas.anyNumber.optional(),
         maxRequestsPerMinute: schemas.anyNumber
             .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
             .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -295,15 +294,18 @@ export class BasicCrawler {
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
+        minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
         if (concurrencySystem !== undefined &&
-            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
-            throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
-                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
+            (minConcurrency !== undefined ||
+                maxConcurrency !== undefined ||
+                initialConcurrency !== undefined ||
+                maxRequestsPerMinute !== undefined)) {
+            throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+                'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+                'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
         }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -370,10 +372,10 @@ export class BasicCrawler {
             this.#failedRequestHandler = failedRequestHandler;
             this.#errorHandler = errorHandler;
             if (requestHandlerTimeoutSecs) {
-                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+                this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             }
             else {
-                this.requestHandlerTimeoutMillis = 60_000;
+                this.#requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
             this.#respectRobotsTxtFile = respectRobotsTxtFile;
@@ -385,7 +387,7 @@ export class BasicCrawler {
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
-                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+                    Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
             this.#maxRequestRetries = maxRequestRetries;
             this.#maxCrawlDepth = maxCrawlDepth;
             this.#sameDomainDelaySecs = sameDomainDelaySecs;
@@ -412,10 +414,10 @@ export class BasicCrawler {
             }));
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
             const maxSignedInteger = 2 ** 31 - 1;
-            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-                this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+            if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
+                this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
                     ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
-                this.requestHandlerTimeoutMillis = maxSignedInteger;
+                this.#requestHandlerTimeoutMillis = maxSignedInteger;
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
             this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
@@ -525,11 +527,14 @@ export class BasicCrawler {
                 },
                 log: this.log,
             };
-            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
             this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
+                // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+                // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+                ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
                 log: this.log,
             }));
         }
@@ -539,7 +544,7 @@ export class BasicCrawler {
     }
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -975,7 +980,7 @@ export class BasicCrawler {
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
         const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
-        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
         await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
     /**
@@ -1167,6 +1172,7 @@ export class BasicCrawler {
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
     async pushData(data, datasetIdentifier) {
+        tryCancel();
         const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
@@ -1256,7 +1262,7 @@ export class BasicCrawler {
         this.#concurrencySystemDep = this.#resolveConcurrencySystem();
         await this.#concurrencySystemDep.ifOwned((system) => system.start());
         this.#autoscaledPool = new AutoscaledPool({
-            ...this.taskLoopOptions,
+            ...this.#taskLoopOptions,
             concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.#identity,
         });
@@ -1289,7 +1295,7 @@ export class BasicCrawler {
      * @param label The request's route label, or `undefined` for the default route / no specific request.
      * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
+    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
         return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.143",
+    "version": "4.0.0-beta.145",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.143",
-        "@crawlee/http-client": "4.0.0-beta.143",
-        "@crawlee/types": "4.0.0-beta.143",
-        "@crawlee/utils": "4.0.0-beta.143",
+        "@crawlee/core": "4.0.0-beta.145",
+        "@crawlee/http-client": "4.0.0-beta.145",
+        "@crawlee/types": "4.0.0-beta.145",
+        "@crawlee/utils": "4.0.0-beta.145",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.143"
+        "@crawlee/impit-client": "^4.0.0-beta.145"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "d44b4c37b5acd824c2093542f878266feb6214ea"
+    "gitHead": "e8b4d50265b9e433ae61c69d16cc1fee00a4d7e9"
 }