@crawlee/basic 4.0.0-beta.123 → 4.0.0-beta.125

package/internals/basic-crawler.d.ts

@@ -29,7 +29,7 @@ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlin
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -137,7 +137,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxRequestRetries?: number;
     /**
-     * Indicates how much time (in seconds) to wait before crawling another same domain request.
+     * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
+     * paced together with the site they belong to.
+     *
+     * Wraps the crawler's request manager in a {@link ThrottlingRequestManager}; pass one as `requestManager`
+     * yourself to configure it further.
      * @default 0
      */
     sameDomainDelaySecs?: number;
@@ -274,10 +278,28 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     onSkippedRequest?: SkippedRequestCallback;
     /**
      * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
-     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} (subclass it to track
-     * extra fields) or any object implementing {@link IStatistics}.
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
+     * implementing {@link IStatistics}.
+     *
+     * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
+     * {@link BasicCrawler.stats|`crawler.stats.state`}:
+     *
+     * ```ts
+     * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
+     *
+     * const crawler = new BasicCrawler({
+     *     statistics,
+     *     requestHandler: async () => {
+     *         statistics.state.productsFound++;
+     *     },
+     * });
+     *
+     * await crawler.run();
+     * // the custom fields are typed on `crawler.stats` too
+     * console.log(crawler.stats.state.productsFound);
+     * ```
      */
-    statistics?: IStatistics;
+    statistics?: IStatistics<StatisticStateExtension>;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
      * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
@@ -429,7 +451,7 @@ interface CrawlerIdentity {
     /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
     readonly hasExplicitId: boolean;
 }
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
     /**
@@ -442,7 +464,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    get stats(): IStatistics;
+    get stats(): IStatistics<StatisticStateExtension>;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
@@ -605,7 +627,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
      * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
@@ -824,12 +846,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Fetches the next request to process from the underlying request provider.
      */
     private fetchNextRequest;
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    private delayRequest;
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     private handleRequest;
     /**

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -198,8 +198,7 @@ export class BasicCrawler {
     internalTimeoutMillis;
     maxRequestRetries;
     maxCrawlDepth;
-    #sameDomainDelayMillis;
-    #domainAccessedTime;
+    #sameDomainDelaySecs;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
@@ -335,6 +334,12 @@ export class BasicCrawler {
                 if (requestList !== undefined || requestQueue !== undefined) {
                     throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
                 }
+                // Both would pace the same domains, from different keys and with no idea of one another.
+                if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
+                    throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
+                        'per domain on its own. Configure the delay on the manager instead, via the ' +
+                        '`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
+                }
                 this.requestManager = requestManager;
             }
             else if (requestList !== undefined && requestQueue !== undefined) {
@@ -355,7 +360,6 @@ export class BasicCrawler {
             this.proxyConfiguration = proxyConfiguration;
             this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
             this.#statusMessageCallback = statusMessageCallback;
-            this.#domainAccessedTime = new Map();
             this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
@@ -382,8 +386,12 @@ export class BasicCrawler {
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
-            this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
+            this.#sameDomainDelaySecs = sameDomainDelaySecs;
+            this.#statsDep = OwnedOrInjected.resolve(statistics, 
+            // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
+            // only be satisfied by an injected instance carrying the matching `state`, so this branch does
+            // not run in that case - hence the cast.
+            () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,
@@ -422,7 +430,7 @@ export class BasicCrawler {
                     if (!source)
                         throw new Error('Request provider is not initialized!');
                     const request = await this.resolveRequest();
-                    if (!request || this.delayRequest(request, source)) {
+                    if (!request) {
                         return;
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
@@ -767,8 +775,14 @@ export class BasicCrawler {
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
             const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
-            if (managerToPurge?.purge && shouldPurge) {
-                await managerToPurge.purge();
+            if (shouldPurge) {
+                await managerToPurge?.purge?.();
+                // The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
+                // sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
+                // the wrapper itself has already covered them.
+                if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
+                    await this.requestManager.purgeDomainQueues();
+                }
             }
             // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
             await this.#statsDep.ifOwned(async (stats) => {
@@ -906,6 +920,19 @@ export class BasicCrawler {
         if (!this.requestManager) {
             this.requestManager = await this.openOwnedRequestQueue();
         }
+        // Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
+        // this point - and because everything that enqueues goes through here first, so nothing slips past the
+        // wrapper into the queue it hides.
+        if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
+            this.requestManager = new ThrottlingRequestManager({
+                inner: this.requestManager,
+                domains: 'all',
+                minCrawlDelaySecs: this.#sameDomainDelaySecs,
+                // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
+                throttleBy: 'registrableDomain',
+                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
+            });
+        }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
@@ -1431,30 +1458,6 @@ export class BasicCrawler {
         }
         return this.requestManager.fetchNextRequest();
     }
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    delayRequest(request, source) {
-        const domain = getDomain(request.url);
-        if (!domain || !request) {
-            return false;
-        }
-        const now = Date.now();
-        const lastAccessTime = this.#domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {
-            this.#domainAccessedTime.set(domain, now);
-            return false;
-        }
-        const delay = lastAccessTime + this.#sameDomainDelayMillis - now;
-        this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
-        setTimeout(async () => {
-            this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
-        }, delay);
-        return true;
-    }
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.123",
+    "version": "4.0.0-beta.125",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.123",
-        "@crawlee/http-client": "4.0.0-beta.123",
-        "@crawlee/types": "4.0.0-beta.123",
-        "@crawlee/utils": "4.0.0-beta.123",
+        "@crawlee/core": "4.0.0-beta.125",
+        "@crawlee/http-client": "4.0.0-beta.125",
+        "@crawlee/types": "4.0.0-beta.125",
+        "@crawlee/utils": "4.0.0-beta.125",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.123"
+        "@crawlee/impit-client": "^4.0.0-beta.125"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "f77648095c6a3f5ed8815c7620ea765db430ae44"
+    "gitHead": "04a7212dd4abe4a5f94f5fc9632acb6089819c8b"
 }