@crawlee/basic 4.0.0-beta.142 → 4.0.0-beta.144

package/internals/basic-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopOptions, TypedRequestsLike } from '@crawlee/core';
 import { ConcurrencySystem, Configuration, ContextPipeline, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
 import { BaseHttpClient } from '@crawlee/http-client';
 import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
@@ -165,7 +165,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
      * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -430,18 +430,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     hasFinishedBefore: boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
-    private requestHandlerTimeoutMillis;
     protected readonly internalTimeoutMillis: number;
     private get handledRequestsCount();
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    /**
-     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
-     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
-     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
-     * settable here.
-     */
-    private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
     /**

package/internals/basic-crawler.js

@@ -7,7 +7,7 @@ import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
@@ -193,8 +193,7 @@ export class BasicCrawler {
     requestHandler;
     #errorHandler;
     #failedRequestHandler;
-    // kept as TS-private: tests read it at runtime
-    requestHandlerTimeoutMillis;
+    #requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     #maxRequestRetries;
     #maxCrawlDepth;
@@ -214,8 +213,7 @@ export class BasicCrawler {
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
-    // kept as TS-private: tests mutate it at runtime
-    taskLoopOptions;
+    #taskLoopOptions;
     httpClient;
     retryOnBlocked;
     #respectRobotsTxtFile;
@@ -370,10 +368,10 @@ export class BasicCrawler {
             this.#failedRequestHandler = failedRequestHandler;
             this.#errorHandler = errorHandler;
             if (requestHandlerTimeoutSecs) {
-                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+                this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             }
             else {
-                this.requestHandlerTimeoutMillis = 60_000;
+                this.#requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
             this.#respectRobotsTxtFile = respectRobotsTxtFile;
@@ -385,7 +383,7 @@ export class BasicCrawler {
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
-                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+                    Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
             this.#maxRequestRetries = maxRequestRetries;
             this.#maxCrawlDepth = maxCrawlDepth;
             this.#sameDomainDelaySecs = sameDomainDelaySecs;
@@ -412,10 +410,10 @@ export class BasicCrawler {
             }));
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
             const maxSignedInteger = 2 ** 31 - 1;
-            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-                this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+            if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
+                this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
                     ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
-                this.requestHandlerTimeoutMillis = maxSignedInteger;
+                this.#requestHandlerTimeoutMillis = maxSignedInteger;
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
             this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
@@ -525,7 +523,7 @@ export class BasicCrawler {
                 },
                 log: this.log,
             };
-            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
             this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
@@ -975,7 +973,7 @@ export class BasicCrawler {
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
         const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
-        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
         await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
     /**
@@ -1167,6 +1165,7 @@ export class BasicCrawler {
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
     async pushData(data, datasetIdentifier) {
+        tryCancel();
         const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
@@ -1256,7 +1255,7 @@ export class BasicCrawler {
         this.#concurrencySystemDep = this.#resolveConcurrencySystem();
         await this.#concurrencySystemDep.ifOwned((system) => system.start());
         this.#autoscaledPool = new AutoscaledPool({
-            ...this.taskLoopOptions,
+            ...this.#taskLoopOptions,
             concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.#identity,
         });
@@ -1289,7 +1288,7 @@ export class BasicCrawler {
      * @param label The request's route label, or `undefined` for the default route / no specific request.
      * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
+    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
         return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.142",
+    "version": "4.0.0-beta.144",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.142",
-        "@crawlee/http-client": "4.0.0-beta.142",
-        "@crawlee/types": "4.0.0-beta.142",
-        "@crawlee/utils": "4.0.0-beta.142",
+        "@crawlee/core": "4.0.0-beta.144",
+        "@crawlee/http-client": "4.0.0-beta.144",
+        "@crawlee/types": "4.0.0-beta.144",
+        "@crawlee/utils": "4.0.0-beta.144",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.142"
+        "@crawlee/impit-client": "^4.0.0-beta.144"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "d40a48fcb78865bd622a05027103155ac08a227f"
+    "gitHead": "4acf68a225909523047f872cbdeb87428bfc560d"
 }