@crawlee/utils 4.0.0-beta.111 → 4.0.0-beta.112

package/internals/robots.d.ts

@@ -44,6 +44,11 @@ export declare class RobotsTxtFile {
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
     private static load;
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent?: string): number | undefined;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/internals/robots.js

@@ -70,11 +70,21 @@ export class RobotsTxtFile {
                 getSitemaps() {
                     return [];
                 },
+                getCrawlDelay() {
+                    return undefined;
+                },
             }, proxyUrl, logger);
         }
         // @ts-ignore
         return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent = '*') {
+        return this.#robots.getCrawlDelay(userAgent);
+    }
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.111",
+    "version": "4.0.0-beta.112",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,8 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.111",
-        "@crawlee/types": "4.0.0-beta.111",
+        "@crawlee/http-client": "4.0.0-beta.112",
+        "@crawlee/types": "4.0.0-beta.112",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "a6b7dec651fa7d81c61689e0a3cd0a8d06fdb586"
+    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
 }