@crawlee/basic 4.0.0-beta.188 → 4.0.0-beta.189

package/internals/autoscaling/concurrency_system.d.ts

@@ -163,7 +163,8 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
     private readonly desiredConcurrencyRatio;
     private readonly scaleUpStepRatio;
     private readonly scaleDownStepRatio;
-    private readonly maxTasksPerMinute;
+    /** The cap on tasks started per minute, or `Infinity` when uncapped. */
+    readonly maxTasksPerMinute: number;
     private readonly snapshotter;
     private readonly systemStatus;
     constructor(options?: ConcurrencySystemOptions);

package/internals/autoscaling/concurrency_system.js

@@ -50,6 +50,7 @@ export class ConcurrencySystem {
     scaleDownStepRatio;
     #loggingIntervalMillis;
     #autoscaleIntervalMillis;
+    /** The cap on tasks started per minute, or `Infinity` when uncapped. */
     maxTasksPerMinute;
     #minConcurrency;
     #maxConcurrency;

package/internals/basic-crawler.d.ts

@@ -386,12 +386,6 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
 }
 export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
-    /**
-     * Tracks the number of crawler instances created. The first crawler uses the default
-     * request queue; subsequent ones get their own queue via a unique alias so they don't
-     * collide.
-     */
-    private static instanceCount;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.

package/internals/basic-crawler.js

@@ -90,8 +90,7 @@ export class BasicCrawler {
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
-    // kept as TS-private: tests reset the counter at runtime
-    static instanceCount = 0;
+    static #instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
@@ -344,7 +343,7 @@ export class BasicCrawler {
             this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
             // Initialize the Configuration instance to avoid lazy loading in the components
             serviceLocator.getConfiguration();
-            const instanceIndex = BasicCrawler.instanceCount++;
+            const instanceIndex = BasicCrawler.#instanceCount++;
             this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
             if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
                 throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
@@ -1525,6 +1524,7 @@ export class BasicCrawler {
         return this.requestManager.fetchNextRequest();
     }
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
         // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
@@ -1662,6 +1662,7 @@ export class BasicCrawler {
      *
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async requestFunctionErrorHandler(error, crawlingContext, request, source) {
         if (error instanceof RequestThrottledError) {
             // The domain told us to come back later, so the request was never really attempted. Put it back
@@ -1720,6 +1721,7 @@ export class BasicCrawler {
         this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;

package/internals/throttling_request_manager.d.ts

@@ -151,8 +151,6 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
 export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
     #private;
     private readonly config;
-    private readonly domainStates;
-    private readonly log;
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
     /**
      * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`

package/internals/throttling_request_manager.js

@@ -105,9 +105,8 @@ export class ThrottlingRequestManager {
     #maxThrottledDomains;
     #persistStateKey;
     #subManagers = new Map();
-    // Not `#private`, unlike the rest: the tests reach for these two.
-    domainStates = new Map();
-    log;
+    #domainStates = new Map();
+    #log;
     /** Domains from the `domains` option, which are throttled whether or not the crawl ever visits them. */
     #listedDomains = new Set();
     /**
@@ -169,7 +168,7 @@ export class ThrottlingRequestManager {
         this.#throttleBy = options.throttleBy ?? 'hostname';
         this.#maxThrottledDomains = options.maxThrottledDomains ?? 100;
         this.#persistStateKey = options.persistStateKey ?? DEFAULT_PERSIST_STATE_KEY;
-        this.log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
+        this.#log = serviceLocator.getLogger().child({ prefix: 'ThrottlingRequestManager' });
         for (const domain of Array.isArray(options.domains) ? options.domains : []) {
             let hostname;
             try {
@@ -182,7 +181,7 @@ export class ThrottlingRequestManager {
             }
             const key = this.#domainKey(hostname);
             this.#listedDomains.add(key);
-            this.domainStates.set(key, newDomainState(key));
+            this.#domainStates.set(key, newDomainState(key));
         }
     }
     /**
@@ -239,7 +238,7 @@ export class ThrottlingRequestManager {
             return;
         }
         this.#warnedAbout.add(key);
-        this.log.warning(message);
+        this.#log.warning(message);
     }
     #extractDomain(url) {
         try {
@@ -251,7 +250,7 @@ export class ThrottlingRequestManager {
     }
     #getDomainState(url) {
         const domain = this.#extractDomain(url);
-        return this.domainStates.get(domain) ?? null;
+        return this.#domainStates.get(domain) ?? null;
     }
     /**
      * The manager that owns a URL's requests, opening a sub-queue for its domain if this is the first time we
@@ -307,10 +306,10 @@ export class ThrottlingRequestManager {
         return subManager;
     }
     #ensureDomainState(domain) {
-        let state = this.domainStates.get(domain);
+        let state = this.#domainStates.get(domain);
         if (!state) {
             state = newDomainState(domain);
-            this.domainStates.set(domain, state);
+            this.#domainStates.set(domain, state);
         }
         return state;
     }
@@ -353,7 +352,7 @@ export class ThrottlingRequestManager {
      */
     #fetchableDomains() {
         const now = Date.now();
-        return Array.from(this.domainStates.values())
+        return Array.from(this.#domainStates.values())
             .filter((state) => now >= throttledUntil(state) && this.#subManagers.has(state.domain))
             .sort((a, b) => throttledUntil(a) - throttledUntil(b))
             .map((state) => state.domain);
@@ -397,7 +396,7 @@ export class ThrottlingRequestManager {
                 `on this manager yourself via \`minCrawlDelaySecs\`.`);
         }
         this.#minCrawlDelayMs = Math.max(this.#minCrawlDelayMs, intervalMs);
-        this.log.debug(`Crawl-delay floor for every domain set to ${(this.#minCrawlDelayMs / 1000).toFixed(1)}s`);
+        this.#log.debug(`Crawl-delay floor for every domain set to ${(this.#minCrawlDelayMs / 1000).toFixed(1)}s`);
         return true;
     }
     /**
@@ -416,7 +415,7 @@ export class ThrottlingRequestManager {
         }
         if (scope === 'hostname' && this.#throttleBy === 'registrableDomain') {
             // Only debug: grouping by registrable domain deliberately paces whole sites, subdomains included.
-            this.log.debug(`Applying a pacing signal scoped to "hostname" across the whole registrable domain, because that ` +
+            this.#log.debug(`Applying a pacing signal scoped to "hostname" across the whole registrable domain, because that ` +
                 `is how this manager groups requests (\`throttleBy\`). Sibling subdomains are paced with it.`);
             return;
         }
@@ -458,14 +457,14 @@ export class ThrottlingRequestManager {
         let delayMs = waitGiven ? waitMs : this.#baseDelayMs * Math.pow(2, state.consecutive429Count - 1);
         if (delayMs > this.#maxDelayMs) {
             const source = waitGiven ? 'requested wait' : 'exponential backoff';
-            this.log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
+            this.#log.warning(`Capping ${source} delay of ${(delayMs / 1000).toFixed(1)}s for domain "${state.domain}" ` +
                 `to maxDelaySecs (${(this.#maxDelayMs / 1000).toFixed(1)}s); the domain may continue to rate-limit. ` +
                 `Consider increasing maxDelaySecs if this recurs.`);
             delayMs = this.#maxDelayMs;
         }
         state.backoffUntil = now + delayMs;
         state.backoffDecaysAt = state.backoffUntil + delayMs;
-        this.log.info(`Rate limit (429) detected for domain "${state.domain}" ` +
+        this.#log.info(`Rate limit (429) detected for domain "${state.domain}" ` +
             `(consecutive: ${state.consecutive429Count}, delay: ${(delayMs / 1000).toFixed(1)}s)`);
         return true;
     }
@@ -487,7 +486,7 @@ export class ThrottlingRequestManager {
         const state = this.#ensureDomainState(domain);
         if (state.declaredCrawlDelayMs === null) {
             state.declaredCrawlDelayMs = intervalMs;
-            this.log.debug(`Set crawl-delay for domain "${state.domain}" to ${(intervalMs / 1000).toFixed(1)}s`);
+            this.#log.debug(`Set crawl-delay for domain "${state.domain}" to ${(intervalMs / 1000).toFixed(1)}s`);
         }
         return true;
     }
@@ -615,7 +614,7 @@ export class ThrottlingRequestManager {
         const dispatchable = [];
         let readyAt;
         let stallCandidates;
-        for (const state of this.domainStates.values()) {
+        for (const state of this.#domainStates.values()) {
             // A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
             // to fetch from and nothing to wait for until then.
             if (!this.#subManagers.has(state.domain)) {
@@ -691,7 +690,7 @@ export class ThrottlingRequestManager {
         const subManagers = await this.#getSubManagers();
         await Promise.all(subManagers.map(async (manager) => manager.purge?.()));
         this.#migratedFromInner = 0;
-        for (const state of this.domainStates.values()) {
+        for (const state of this.#domainStates.values()) {
             state.consecutive429Count = 0;
             state.backoffUntil = 0;
             state.crawlDelayUntil = 0;
@@ -734,7 +733,7 @@ export class ThrottlingRequestManager {
     async fetchNextRequest() {
         await this.#ensureSubManagers();
         for (const domain of this.#fetchableDomains()) {
-            const state = this.domainStates.get(domain);
+            const state = this.#domainStates.get(domain);
             // Armed while the fetch below is still suspended, so that a concurrent `fetchNextRequest` cannot
             // find the domain fetchable and dispatch into the same window - which would pace each task
             // rather than the domain.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.188",
+    "version": "4.0.0-beta.189",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.188",
-        "@crawlee/http-client": "4.0.0-beta.188",
-        "@crawlee/types": "4.0.0-beta.188",
-        "@crawlee/utils": "4.0.0-beta.188",
+        "@crawlee/core": "4.0.0-beta.189",
+        "@crawlee/http-client": "4.0.0-beta.189",
+        "@crawlee/types": "4.0.0-beta.189",
+        "@crawlee/utils": "4.0.0-beta.189",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.188"
+        "@crawlee/impit-client": "4.0.0-beta.189"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "e328abb1ed78f0f4521fe12d7c1cb63a922a38f8"
+    "gitHead": "1dbb0f272b9da9c7a6bf3bdc1ae48f84eda3f21c"
 }