@crawlee/basic 4.0.0-beta.111 → 4.0.0-beta.112

package/internals/basic-crawler.d.ts

@@ -237,6 +237,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     statusMessageCallback?: StatusMessageCallback;
     /**
      * HTTP status codes that indicate the session should be retired.
+     *
+     * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
+     * this is consulted, so removing 429 here only affects domains that manager does not cover.
+     *
      * @default [401, 403, 429]
      */
     blockedStatusCodes?: number[];
@@ -518,6 +522,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         requestList: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestQueue: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        requestManager: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -778,6 +784,21 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     protected throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
+    /**
+     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
+     *
+     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
+     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
+     */
+    private applyCrawlDelay;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
     private pauseOnMigration;
     /**
@@ -836,6 +857,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @returns The message to be logged
      */
     protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    private errorAbsolvesSession;
     private canRequestBeRetried;
     /**
      * Stops the crawler immediately.

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -204,6 +204,7 @@ export class BasicCrawler {
         extendContext: ow.optional.function,
         requestList: ow.optional.object.validate(validators.requestList),
         requestQueue: ow.optional.object.validate(validators.requestQueue),
+        requestManager: ow.optional.object,
         // Subclasses override this function instead of passing it
         // in constructor, so this validation needs to apply only
         // if the user creates an instance of BasicCrawler directly.
@@ -419,7 +420,7 @@ export class BasicCrawler {
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
                             // SessionError already retired the session in `requestFunctionErrorHandler`;
                             // skip `markBad` to avoid double-counting usage/error score.
-                            if (!(unwrappedError instanceof SessionError)) {
+                            if (!this.errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
                             return;
@@ -456,6 +457,11 @@ export class BasicCrawler {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
+                    // Checked here because this runs only once nothing is in flight, which is exactly when a
+                    // crawl that cannot progress looks indistinguishable from one that is merely waiting.
+                    if (!keepAlive && supportsDomainThrottling(this.requestManager)) {
+                        await this.requestManager.assertNoStalledDomains();
+                    }
                     const isFinished = isFinishedFunction
                         ? await isFinishedFunction()
                         : await this.defaultIsFinishedFunction();
@@ -974,9 +980,9 @@ export class BasicCrawler {
             await this.onSkippedRequest?.(options);
         });
     }
-    logOncePerRun(key, message) {
+    logOncePerRun(key, message, level = 'info') {
         if (!this.#loggedPerRun.has(key)) {
-            this.log.info(message);
+            this.log[level](message);
             this.#loggedPerRun.add(key);
         }
     }
@@ -1240,8 +1246,48 @@ export class BasicCrawler {
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
         const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
+        if (robotsTxtFile) {
+            const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
+            if (crawlDelay !== undefined) {
+                this.applyCrawlDelay(url, crawlDelay);
+            }
+        }
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    recordDomainRateLimit(url, retryAfterHeader) {
+        if (supportsDomainThrottling(this.requestManager) &&
+            this.requestManager.recordDomainDelay(url, parseRetryAfterHeader(retryAfterHeader))) {
+            return true;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but nothing is set up to back off from it, ` +
+            'so the response is handled like any other, with no per-domain delay. ' +
+            `Pass a \`ThrottlingRequestManager\` as \`requestManager\` and include "${domain}" in its \`domains\` ` +
+            'option to honour `Retry-After` and apply exponential backoff instead.', 'warning');
+        return false;
+    }
+    /**
+     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
+     *
+     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
+     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
+     */
+    applyCrawlDelay(url, delaySeconds) {
+        if (supportsDomainThrottling(this.requestManager) && this.requestManager.setCrawlDelay(url, delaySeconds)) {
+            return;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but nothing is set up to honour it, ` +
+            'so requests to that domain will not be paced. Pass a `ThrottlingRequestManager` as `requestManager` ' +
+            `and include "${domain}" in its \`domains\` option to enforce the delay.`, 'warning');
+    }
     async getRobotsTxtFileForUrl(url) {
         if (!this.#respectRobotsTxtFile) {
             return undefined;
@@ -1382,7 +1428,7 @@ export class BasicCrawler {
             }
             // decrease the session score if the request fails (but the error handler did not throw);
             // skip when the error is a SessionError, which already retired the session
-            if (!(err instanceof SessionError)) {
+            if (!this.errorAbsolvesSession(err)) {
                 crawlingContext.session.markBad();
             }
         }
@@ -1508,6 +1554,16 @@ export class BasicCrawler {
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
     async requestFunctionErrorHandler(error, crawlingContext, request, source) {
+        if (error instanceof RequestThrottledError) {
+            // The domain told us to come back later, so the request was never really attempted. Put it back
+            // without recording a failure - it costs neither a retry nor session reputation.
+            this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
+                id: request.id,
+                url: request.url,
+            });
+            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            return;
+        }
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
@@ -1584,6 +1640,13 @@ export class BasicCrawler {
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
             : [error.message || error, userLine].join('\n');
     }
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    errorAbsolvesSession(error) {
+        return error instanceof SessionError || error instanceof RequestThrottledError;
+    }
     canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
@@ -1669,6 +1732,10 @@ export class BasicCrawler {
         }
     }
 }
+/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
+function hostnameOrUrl(url) {
+    return URL.canParse(url) ? new URL(url).hostname : url;
+}
 export function createBasicRouter(routes) {
     return Router.create(routes);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.111",
+    "version": "4.0.0-beta.112",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.111",
-        "@crawlee/http-client": "4.0.0-beta.111",
-        "@crawlee/types": "4.0.0-beta.111",
-        "@crawlee/utils": "4.0.0-beta.111",
+        "@crawlee/core": "4.0.0-beta.112",
+        "@crawlee/http-client": "4.0.0-beta.112",
+        "@crawlee/types": "4.0.0-beta.112",
+        "@crawlee/utils": "4.0.0-beta.112",
         "csv-stringify": "^6.5.2",
         "ow": "^2.0.0",
         "tldts": "^7.0.6",
@@ -53,7 +53,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.111"
+        "@crawlee/impit-client": "^4.0.0-beta.112"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "a6b7dec651fa7d81c61689e0a3cd0a8d06fdb586"
+    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
 }