@crawlee/utils 4.0.0-beta.110 → 4.0.0-beta.112

package/index.d.ts

@@ -1,10 +1,8 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
+export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';

package/index.js

@@ -1,10 +1,7 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { sleep, expandShadowRoots } from './internals/general.js';
 export * as social from './internals/social.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';

package/internal.d.ts

@@ -0,0 +1,7 @@
+export * from './internals/blocked.js';
+export type { CheerioRoot, CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';

package/internal.js

@@ -0,0 +1,6 @@
+export * from './internals/blocked.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';

package/internals/robots.d.ts

@@ -44,6 +44,11 @@ export declare class RobotsTxtFile {
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
     private static load;
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent?: string): number | undefined;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/internals/robots.js

@@ -70,11 +70,21 @@ export class RobotsTxtFile {
                 getSitemaps() {
                     return [];
                 },
+                getCrawlDelay() {
+                    return undefined;
+                },
             }, proxyUrl, logger);
         }
         // @ts-ignore
         return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent = '*') {
+        return this.#robots.getCrawlDelay(userAgent);
+    }
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.110",
+    "version": "4.0.0-beta.112",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -8,6 +8,7 @@
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -42,8 +43,8 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.110",
-        "@crawlee/types": "4.0.0-beta.110",
+        "@crawlee/http-client": "4.0.0-beta.112",
+        "@crawlee/types": "4.0.0-beta.112",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
@@ -61,5 +62,5 @@
             }
         }
     },
-    "gitHead": "71f42c48f431ca46b04491eca65b191691e75033"
+    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
 }