@crawlee/utils 4.0.0-beta.11 → 4.0.0-beta.111

package/internals/robots.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
@@ -18,15 +19,23 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export declare class RobotsTxtFile {
-    private robots;
-    private proxyUrl?;
+    #private;
     private constructor();
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
      * @param url the URL to fetch robots.txt for
-     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+     * @param [options] additional options
+     * @param [options.signal] an AbortSignal to cancel the request
+     * @param [options.timeoutMillis] timeout in milliseconds for the request
+     * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static find(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    static find(url: string, options?: {
+        signal?: AbortSignal;
+        timeoutMillis?: number;
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+        logger?: CrawleeLogger;
+    }): Promise<RobotsTxtFile>;
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
      * @param url the URL for robots.txt file
@@ -34,7 +43,7 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    protected static load(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    private static load;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt
@@ -54,5 +63,3 @@ export declare class RobotsTxtFile {
      */
     parseUrlsFromSitemaps(): Promise<string[]>;
 }
-export { RobotsTxtFile as RobotsFile };
-//# sourceMappingURL=robots.d.ts.map

package/internals/robots.js

@@ -1,7 +1,6 @@
-import { gotScraping } from 'got-scraping';
+import { FetchHttpClient } from '@crawlee/http-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
-let HTTPError;
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -21,22 +20,27 @@ let HTTPError;
  * ```
  */
 export class RobotsTxtFile {
-    robots;
-    proxyUrl;
-    constructor(robots, proxyUrl) {
-        this.robots = robots;
-        this.proxyUrl = proxyUrl;
+    #robots;
+    #proxyUrl;
+    #logger;
+    constructor(robots, proxyUrl, logger) {
+        this.#robots = robots;
+        this.#proxyUrl = proxyUrl;
+        this.#logger = logger;
     }
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
      * @param url the URL to fetch robots.txt for
-     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+     * @param [options] additional options
+     * @param [options.signal] an AbortSignal to cancel the request
+     * @param [options.timeoutMillis] timeout in milliseconds for the request
+     * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static async find(url, proxyUrl) {
+    static async find(url, options) {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), proxyUrl);
+        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -48,33 +52,28 @@ export class RobotsTxtFile {
         // @ts-ignore
         return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
     }
-    static async load(url, proxyUrl) {
-        if (!HTTPError) {
-            HTTPError = (await import('got-scraping')).HTTPError;
-        }
-        try {
-            const response = await gotScraping({
-                url,
-                proxyUrl,
-                method: 'GET',
-                responseType: 'text',
-            });
-            // @ts-ignore
-            return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
+    static async load(url, options) {
+        const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
+        const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
+            proxyUrl,
+            timeoutMillis: options?.timeoutMillis,
+            signal: options?.signal,
+        });
+        if (response.status < 200 || response.status >= 300) {
+            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
         }
-        catch (e) {
-            if (e instanceof HTTPError && e.response.statusCode === 404) {
-                return new RobotsTxtFile({
-                    isAllowed() {
-                        return true;
-                    },
-                    getSitemaps() {
-                        return [];
-                    },
-                }, proxyUrl);
-            }
-            throw e;
+        if (response.status === 404) {
+            return new RobotsTxtFile({
+                isAllowed() {
+                    return true;
+                },
+                getSitemaps() {
+                    return [];
+                },
+            }, proxyUrl, logger);
         }
+        // @ts-ignore
+        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }
     /**
      * Check if a URL should be crawled by robots.
@@ -82,19 +81,19 @@ export class RobotsTxtFile {
      * @param [userAgent] relevant user agent, default to `*`
      */
     isAllowed(url, userAgent = '*') {
-        return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
+        return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
     }
     /**
      * Get URLs of sitemaps referenced in the robots file.
      */
     getSitemaps() {
-        return this.robots.getSitemaps();
+        return this.#robots.getSitemaps();
     }
     /**
      * Parse all the sitemaps referenced in the robots file.
      */
     async parseSitemaps() {
-        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
+        return Sitemap.load(this.#robots.getSitemaps(), this.#proxyUrl, { logger: this.#logger });
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
@@ -103,6 +102,3 @@ export class RobotsTxtFile {
         return (await this.parseSitemaps()).urls;
     }
 }
-// to stay backwards compatible
-export { RobotsTxtFile as RobotsFile };
-//# sourceMappingURL=robots.js.map

package/internals/sitemap.d.ts

@@ -1,5 +1,4 @@
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { Delays } from 'got-scraping';
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -36,9 +35,29 @@ export interface ParseSitemapOptions {
      */
     sitemapRetries?: number;
     /**
-     * Network timeouts for sitemap fetching. See [Got documentation](https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md) for more details.
+     * Timeout settings for network requests when fetching sitemaps. By default this is `30000` milliseconds (30 seconds).
      */
-    networkTimeouts?: Delays;
+    timeoutMillis?: number;
+    /**
+     * If true, the parser will log a warning if it fails to fetch a sitemap due to a network error
+     * @default true
+     */
+    reportNetworkErrors?: boolean;
+    /**
+     * Custom HTTP client to be used for fetching sitemaps.
+     */
+    httpClient?: BaseHttpClient;
+    /**
+     * Optional filter for nested sitemap URLs discovered in sitemap index files.
+     * Called with the URL of each child sitemap before it is fetched.
+     * Return `true` to include the sitemap, `false` to skip it.
+     * If not provided, all nested sitemaps are followed.
+     */
+    nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Optional logger for reporting warnings during sitemap parsing.
+     */
+    logger?: CrawleeLogger;
 }
 export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
 /**
@@ -62,7 +81,7 @@ export declare class Sitemap {
      * @param url The domain URL to fetch the sitemap for.
      * @param proxyUrl A proxy to be used for fetching the sitemap file.
      */
-    static tryCommonNames(url: string, proxyUrl?: string): Promise<Sitemap>;
+    static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
     /**
      * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
      * @param urls sitemap URL(s)
@@ -74,8 +93,46 @@ export declare class Sitemap {
      * @param content XML sitemap content
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
-    static fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap>;
-    protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+    static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
+    private static parse;
 }
+/**
+ * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
+ * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
+ * @param `urls` The list of URLs to discover sitemaps for.
+ * @param `options` Options for sitemap discovery
+ * @returns An async iterable with the discovered sitemap URLs.
+ */
+export declare function discoverValidSitemaps(urls: string[], options?: {
+    /**
+     * Proxy URL to be used for network requests.
+     */
+    proxyUrl?: string;
+    /**
+     * Timeout in milliseconds for the entire `discoverValidSitemaps` call.
+     * An `AbortController` is created internally and its signal is passed to every HTTP request,
+     * so the whole discovery operation is cancelled once the timeout elapses.
+     * Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
+     */
+    timeoutMillis?: number;
+    /**
+     * An external `AbortSignal` to cancel the entire discovery operation.
+     * If both `signal` and `timeout` are provided, the operation is cancelled
+     * when either the signal is aborted or the timeout elapses (whichever comes first).
+     */
+    signal?: AbortSignal;
+    /**
+     * Timeout in milliseconds for each individual HTTP request during discovery.
+     * Defaults to `20000` ms (20 seconds).
+     */
+    requestTimeoutMillis?: number;
+    /**
+     * HTTP client to be used for network requests.
+     */
+    httpClient?: BaseHttpClient;
+    /**
+     * Optional logger for reporting warnings during sitemap discovery.
+     */
+    logger?: CrawleeLogger;
+}): AsyncIterable<string>;
 export {};
-//# sourceMappingURL=sitemap.d.ts.map